2024-12-04 14:55:53,015 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-04 14:55:53,035 main DEBUG Took 0.009357 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 14:55:53,036 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 14:55:53,036 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 14:55:53,037 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 14:55:53,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,045 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 14:55:53,055 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,057 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,057 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,057 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,058 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,058 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,059 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,059 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,060 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,060 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,061 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,061 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,061 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,062 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,062 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,062 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,063 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,063 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,063 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,063 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,064 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,064 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,064 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,065 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 14:55:53,065 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,065 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 14:55:53,066 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 14:55:53,068 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 14:55:53,069 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 14:55:53,070 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 14:55:53,071 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 14:55:53,071 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 14:55:53,079 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 14:55:53,081 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 14:55:53,083 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 14:55:53,083 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 14:55:53,083 main DEBUG createAppenders(={Console}) 2024-12-04 14:55:53,084 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-04 14:55:53,084 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-04 14:55:53,085 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-04 14:55:53,085 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 14:55:53,085 main DEBUG OutputStream closed 2024-12-04 14:55:53,086 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 14:55:53,086 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 14:55:53,086 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-04 14:55:53,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 14:55:53,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 14:55:53,157 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 14:55:53,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 14:55:53,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 14:55:53,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 14:55:53,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 14:55:53,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 14:55:53,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 14:55:53,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 14:55:53,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 14:55:53,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 14:55:53,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 14:55:53,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 14:55:53,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 14:55:53,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 14:55:53,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 14:55:53,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 14:55:53,166 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 14:55:53,166 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-04 14:55:53,166 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 14:55:53,167 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-04T14:55:53,368 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7 2024-12-04 14:55:53,371 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 14:55:53,371 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T14:55:53,379 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-12-04T14:55:53,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T14:55:53,413 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/cluster_654e7680-3d13-2f9f-e580-aa09589746a6, deleteOnExit=true 2024-12-04T14:55:53,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T14:55:53,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/test.cache.data in system properties and HBase conf 2024-12-04T14:55:53,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T14:55:53,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/hadoop.log.dir in system properties and HBase conf 2024-12-04T14:55:53,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T14:55:53,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T14:55:53,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T14:55:53,503 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T14:55:53,597 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T14:55:53,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:55:53,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T14:55:53,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T14:55:53,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:55:53,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T14:55:53,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T14:55:53,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T14:55:53,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:55:53,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T14:55:53,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/nfs.dump.dir in system properties and HBase conf 2024-12-04T14:55:53,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/java.io.tmpdir in system properties and HBase conf 2024-12-04T14:55:53,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T14:55:53,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T14:55:53,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T14:55:54,532 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T14:55:54,597 INFO [Time-limited test {}] log.Log(170): Logging initialized @2214ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T14:55:54,657 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:55:54,710 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:55:54,728 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:55:54,728 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:55:54,729 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:55:54,740 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:55:54,742 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:55:54,743 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:55:54,899 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/java.io.tmpdir/jetty-localhost-45729-hadoop-hdfs-3_4_1-tests_jar-_-any-103753389564245673/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:55:54,904 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:45729} 2024-12-04T14:55:54,904 INFO [Time-limited test {}] server.Server(415): Started @2522ms 2024-12-04T14:55:55,465 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T14:55:55,477 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T14:55:55,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T14:55:55,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T14:55:55,479 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T14:55:55,483 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/hadoop.log.dir/,AVAILABLE} 2024-12-04T14:55:55,484 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T14:55:55,642 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/java.io.tmpdir/jetty-localhost-37479-hadoop-hdfs-3_4_1-tests_jar-_-any-4471868447477975218/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:55:55,644 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:37479} 2024-12-04T14:55:55,644 INFO [Time-limited test {}] server.Server(415): Started @3262ms 2024-12-04T14:55:55,701 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T14:55:56,516 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/cluster_654e7680-3d13-2f9f-e580-aa09589746a6/data/data1/current/BP-1812510059-172.17.0.2-1733324154111/current, will proceed with Du for space computation calculation, 2024-12-04T14:55:56,516 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/cluster_654e7680-3d13-2f9f-e580-aa09589746a6/data/data2/current/BP-1812510059-172.17.0.2-1733324154111/current, will proceed with Du for space computation calculation, 2024-12-04T14:55:56,540 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T14:55:56,579 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7 2024-12-04T14:55:56,582 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x146f54dbd02c06d0 with lease ID 0xa8d4088707cbc765: Processing first storage report for DS-4a7129ce-3f0f-4ff0-b959-d3d1a01d4347 from datanode DatanodeRegistration(127.0.0.1:35153, datanodeUuid=d3ec1071-8fa2-49ad-85bb-9771668d1f80, infoPort=46813, infoSecurePort=0, ipcPort=33625, storageInfo=lv=-57;cid=testClusterID;nsid=1374277302;c=1733324154112) 2024-12-04T14:55:56,584 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x146f54dbd02c06d0 with lease ID 0xa8d4088707cbc765: from storage DS-4a7129ce-3f0f-4ff0-b959-d3d1a01d4347 node DatanodeRegistration(127.0.0.1:35153, datanodeUuid=d3ec1071-8fa2-49ad-85bb-9771668d1f80, infoPort=46813, infoSecurePort=0, ipcPort=33625, storageInfo=lv=-57;cid=testClusterID;nsid=1374277302;c=1733324154112), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-04T14:55:56,584 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x146f54dbd02c06d0 with lease ID 0xa8d4088707cbc765: Processing first storage report for DS-040fde0d-b820-4527-8bbe-6c6f49028344 from datanode DatanodeRegistration(127.0.0.1:35153, datanodeUuid=d3ec1071-8fa2-49ad-85bb-9771668d1f80, infoPort=46813, infoSecurePort=0, ipcPort=33625, storageInfo=lv=-57;cid=testClusterID;nsid=1374277302;c=1733324154112) 2024-12-04T14:55:56,584 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x146f54dbd02c06d0 with lease ID 0xa8d4088707cbc765: from storage DS-040fde0d-b820-4527-8bbe-6c6f49028344 node DatanodeRegistration(127.0.0.1:35153, datanodeUuid=d3ec1071-8fa2-49ad-85bb-9771668d1f80, infoPort=46813, infoSecurePort=0, ipcPort=33625, storageInfo=lv=-57;cid=testClusterID;nsid=1374277302;c=1733324154112), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T14:55:56,640 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/cluster_654e7680-3d13-2f9f-e580-aa09589746a6/zookeeper_0, clientPort=63984, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/cluster_654e7680-3d13-2f9f-e580-aa09589746a6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/cluster_654e7680-3d13-2f9f-e580-aa09589746a6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T14:55:56,650 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63984 2024-12-04T14:55:56,659 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:55:56,662 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:55:56,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741825_1001 (size=7) 2024-12-04T14:55:57,270 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1 with version=8 2024-12-04T14:55:57,271 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/hbase-staging 2024-12-04T14:55:57,351 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T14:55:57,577 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d214edae7bd2:0 server-side Connection retries=45 2024-12-04T14:55:57,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:55:57,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:55:57,591 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:55:57,592 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:55:57,592 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:55:57,705 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T14:55:57,753 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T14:55:57,760 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T14:55:57,763 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:55:57,783 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 104217 (auto-detected) 2024-12-04T14:55:57,784 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T14:55:57,800 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45797 2024-12-04T14:55:57,818 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45797 connecting to ZooKeeper ensemble=127.0.0.1:63984 2024-12-04T14:55:57,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:457970x0, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:55:57,928 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45797-0x101a22151460000 connected 2024-12-04T14:55:57,995 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:55:57,998 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:55:58,009 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:55:58,012 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1, hbase.cluster.distributed=false 2024-12-04T14:55:58,032 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:55:58,035 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45797 2024-12-04T14:55:58,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45797 2024-12-04T14:55:58,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45797 2024-12-04T14:55:58,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45797 2024-12-04T14:55:58,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45797 2024-12-04T14:55:58,128 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d214edae7bd2:0 server-side Connection retries=45 2024-12-04T14:55:58,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:55:58,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T14:55:58,130 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T14:55:58,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T14:55:58,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T14:55:58,132 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T14:55:58,134 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T14:55:58,135 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34477 2024-12-04T14:55:58,136 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34477 connecting to ZooKeeper ensemble=127.0.0.1:63984 2024-12-04T14:55:58,137 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:55:58,139 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:55:58,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344770x0, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T14:55:58,153 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34477-0x101a22151460001 connected 2024-12-04T14:55:58,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:55:58,157 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T14:55:58,164 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T14:55:58,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T14:55:58,172 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T14:55:58,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34477 2024-12-04T14:55:58,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34477 2024-12-04T14:55:58,175 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34477 2024-12-04T14:55:58,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34477 2024-12-04T14:55:58,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34477 2024-12-04T14:55:58,191 DEBUG [M:0;d214edae7bd2:45797 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d214edae7bd2:45797 2024-12-04T14:55:58,192 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d214edae7bd2,45797,1733324157426 2024-12-04T14:55:58,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:55:58,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:55:58,204 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d214edae7bd2,45797,1733324157426 2024-12-04T14:55:58,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T14:55:58,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:55:58,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:55:58,235 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T14:55:58,236 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d214edae7bd2,45797,1733324157426 from backup master directory 2024-12-04T14:55:58,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d214edae7bd2,45797,1733324157426 2024-12-04T14:55:58,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:55:58,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T14:55:58,244 WARN [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:55:58,244 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d214edae7bd2,45797,1733324157426 2024-12-04T14:55:58,247 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T14:55:58,248 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T14:55:58,295 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/hbase.id] with ID: f1d58f65-ea39-40cd-ad65-dc760c49aa67 2024-12-04T14:55:58,295 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/.tmp/hbase.id 2024-12-04T14:55:58,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741826_1002 (size=42) 2024-12-04T14:55:58,305 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/.tmp/hbase.id]:[hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/hbase.id] 2024-12-04T14:55:58,348 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T14:55:58,352 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T14:55:58,369 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-04T14:55:58,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:55:58,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:55:58,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741827_1003 (size=196) 2024-12-04T14:55:58,407 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:55:58,409 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T14:55:58,422 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:55:58,425 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T14:55:58,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741828_1004 (size=1189) 2024-12-04T14:55:58,466 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store 2024-12-04T14:55:58,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741829_1005 (size=34) 2024-12-04T14:55:58,891 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T14:55:58,895 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:55:58,897 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:55:58,897 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:55:58,897 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:55:58,899 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:55:58,899 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:55:58,899 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:55:58,901 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733324158896Disabling compacts and flushes for region at 1733324158896Disabling writes for close at 1733324158899 (+3 ms)Writing region close event to WAL at 1733324158899Closed at 1733324158899 2024-12-04T14:55:58,903 WARN [master/d214edae7bd2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/.initializing 2024-12-04T14:55:58,903 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/WALs/d214edae7bd2,45797,1733324157426 2024-12-04T14:55:58,911 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T14:55:58,924 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d214edae7bd2%2C45797%2C1733324157426, suffix=, logDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/WALs/d214edae7bd2,45797,1733324157426, archiveDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/oldWALs, maxLogs=10 2024-12-04T14:55:58,945 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/WALs/d214edae7bd2,45797,1733324157426/d214edae7bd2%2C45797%2C1733324157426.1733324158929, exclude list is [], retry=0 2024-12-04T14:55:58,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35153,DS-4a7129ce-3f0f-4ff0-b959-d3d1a01d4347,DISK] 2024-12-04T14:55:58,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T14:55:58,995 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/WALs/d214edae7bd2,45797,1733324157426/d214edae7bd2%2C45797%2C1733324157426.1733324158929 2024-12-04T14:55:58,996 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46813:46813)] 2024-12-04T14:55:58,996 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:55:58,997 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:55:59,000 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,001 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,033 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T14:55:59,059 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:55:59,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:55:59,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,065 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T14:55:59,066 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:55:59,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:55:59,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,069 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T14:55:59,069 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:55:59,070 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:55:59,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,073 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T14:55:59,073 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:55:59,074 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:55:59,074 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,078 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,079 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,083 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,084 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,088 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T14:55:59,093 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T14:55:59,098 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:55:59,099 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59362312, jitterRate=-0.11543262004852295}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T14:55:59,107 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733324159012Initializing all the Stores at 1733324159014 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324159014Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324159015 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324159015Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324159015Cleaning up temporary data from old regions at 1733324159084 (+69 ms)Region opened successfully at 1733324159107 (+23 ms) 2024-12-04T14:55:59,109 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T14:55:59,145 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5766a8ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d214edae7bd2/172.17.0.2:0 2024-12-04T14:55:59,173 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T14:55:59,183 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T14:55:59,183 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T14:55:59,186 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T14:55:59,187 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T14:55:59,191 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T14:55:59,191 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T14:55:59,216 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T14:55:59,224 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T14:55:59,268 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T14:55:59,271 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T14:55:59,272 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T14:55:59,277 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T14:55:59,279 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T14:55:59,282 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T14:55:59,293 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T14:55:59,295 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T14:55:59,301 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T14:55:59,318 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T14:55:59,326 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T14:55:59,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:55:59,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T14:55:59,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:55:59,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:55:59,339 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d214edae7bd2,45797,1733324157426, sessionid=0x101a22151460000, setting cluster-up flag (Was=false) 2024-12-04T14:55:59,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:55:59,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:55:59,393 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T14:55:59,396 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d214edae7bd2,45797,1733324157426 2024-12-04T14:55:59,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:55:59,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:55:59,443 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T14:55:59,446 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d214edae7bd2,45797,1733324157426 2024-12-04T14:55:59,456 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T14:55:59,484 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(746): ClusterId : f1d58f65-ea39-40cd-ad65-dc760c49aa67 2024-12-04T14:55:59,486 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T14:55:59,496 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T14:55:59,496 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T14:55:59,503 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T14:55:59,503 DEBUG [RS:0;d214edae7bd2:34477 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f07b1b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d214edae7bd2/172.17.0.2:0 2024-12-04T14:55:59,515 DEBUG [RS:0;d214edae7bd2:34477 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d214edae7bd2:34477 2024-12-04T14:55:59,518 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T14:55:59,518 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T14:55:59,518 DEBUG [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T14:55:59,520 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(2659): reportForDuty to master=d214edae7bd2,45797,1733324157426 with port=34477, startcode=1733324158096 2024-12-04T14:55:59,520 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T14:55:59,528 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T14:55:59,529 DEBUG [RS:0;d214edae7bd2:34477 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T14:55:59,533 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T14:55:59,538 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d214edae7bd2,45797,1733324157426 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T14:55:59,545 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d214edae7bd2:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:55:59,546 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d214edae7bd2:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:55:59,546 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d214edae7bd2:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:55:59,547 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d214edae7bd2:0, corePoolSize=5, maxPoolSize=5 2024-12-04T14:55:59,547 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d214edae7bd2:0, corePoolSize=10, maxPoolSize=10 2024-12-04T14:55:59,547 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,548 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d214edae7bd2:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:55:59,548 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,551 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733324189551 2024-12-04T14:55:59,554 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T14:55:59,556 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T14:55:59,556 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:55:59,557 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T14:55:59,560 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T14:55:59,560 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T14:55:59,561 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T14:55:59,561 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T14:55:59,562 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:55:59,562 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T14:55:59,561 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,565 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T14:55:59,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57245, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T14:55:59,567 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T14:55:59,567 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T14:55:59,573 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45797 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:55:59,574 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T14:55:59,575 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T14:55:59,577 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.large.0-1733324159577,5,FailOnTimeoutGroup] 2024-12-04T14:55:59,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741831_1007 (size=1321) 2024-12-04T14:55:59,578 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.small.0-1733324159578,5,FailOnTimeoutGroup] 2024-12-04T14:55:59,578 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,579 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T14:55:59,580 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,580 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,596 DEBUG [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-04T14:55:59,596 WARN [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-04T14:55:59,699 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(2659): reportForDuty to master=d214edae7bd2,45797,1733324157426 with port=34477, startcode=1733324158096 2024-12-04T14:55:59,704 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45797 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d214edae7bd2,34477,1733324158096 2024-12-04T14:55:59,710 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45797 {}] master.ServerManager(517): Registering regionserver=d214edae7bd2,34477,1733324158096 2024-12-04T14:55:59,719 DEBUG [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1 2024-12-04T14:55:59,719 DEBUG [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41043 2024-12-04T14:55:59,719 DEBUG [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T14:55:59,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:55:59,727 DEBUG [RS:0;d214edae7bd2:34477 {}] zookeeper.ZKUtil(111): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d214edae7bd2,34477,1733324158096 2024-12-04T14:55:59,727 WARN [RS:0;d214edae7bd2:34477 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T14:55:59,728 INFO [RS:0;d214edae7bd2:34477 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T14:55:59,728 DEBUG [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/WALs/d214edae7bd2,34477,1733324158096 2024-12-04T14:55:59,730 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d214edae7bd2,34477,1733324158096] 2024-12-04T14:55:59,754 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T14:55:59,765 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T14:55:59,770 INFO [RS:0;d214edae7bd2:34477 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T14:55:59,771 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,772 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T14:55:59,777 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T14:55:59,778 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,778 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,779 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,779 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,779 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,779 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,779 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d214edae7bd2:0, corePoolSize=2, maxPoolSize=2 2024-12-04T14:55:59,780 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,780 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,780 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,780 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,780 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,781 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d214edae7bd2:0, corePoolSize=1, maxPoolSize=1 2024-12-04T14:55:59,781 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d214edae7bd2:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:55:59,781 DEBUG [RS:0;d214edae7bd2:34477 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0, corePoolSize=3, maxPoolSize=3 2024-12-04T14:55:59,782 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,782 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,782 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,782 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,783 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,783 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,34477,1733324158096-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:55:59,805 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T14:55:59,807 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,34477,1733324158096-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,807 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,808 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.Replication(171): d214edae7bd2,34477,1733324158096 started 2024-12-04T14:55:59,825 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:55:59,825 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(1482): Serving as d214edae7bd2,34477,1733324158096, RpcServer on d214edae7bd2/172.17.0.2:34477, sessionid=0x101a22151460001 2024-12-04T14:55:59,826 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T14:55:59,826 DEBUG [RS:0;d214edae7bd2:34477 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d214edae7bd2,34477,1733324158096 2024-12-04T14:55:59,826 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd214edae7bd2,34477,1733324158096' 2024-12-04T14:55:59,827 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T14:55:59,827 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T14:55:59,828 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T14:55:59,828 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T14:55:59,828 DEBUG [RS:0;d214edae7bd2:34477 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d214edae7bd2,34477,1733324158096 2024-12-04T14:55:59,828 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd214edae7bd2,34477,1733324158096' 2024-12-04T14:55:59,829 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T14:55:59,829 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T14:55:59,830 DEBUG [RS:0;d214edae7bd2:34477 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T14:55:59,830 INFO [RS:0;d214edae7bd2:34477 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T14:55:59,830 INFO [RS:0;d214edae7bd2:34477 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T14:55:59,941 INFO [RS:0;d214edae7bd2:34477 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T14:55:59,946 INFO [RS:0;d214edae7bd2:34477 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d214edae7bd2%2C34477%2C1733324158096, suffix=, logDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/WALs/d214edae7bd2,34477,1733324158096, archiveDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/oldWALs, maxLogs=32 2024-12-04T14:55:59,959 DEBUG [RS:0;d214edae7bd2:34477 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/WALs/d214edae7bd2,34477,1733324158096/d214edae7bd2%2C34477%2C1733324158096.1733324159948, exclude list is [], retry=0 2024-12-04T14:55:59,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35153,DS-4a7129ce-3f0f-4ff0-b959-d3d1a01d4347,DISK] 2024-12-04T14:55:59,966 INFO [RS:0;d214edae7bd2:34477 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/WALs/d214edae7bd2,34477,1733324158096/d214edae7bd2%2C34477%2C1733324158096.1733324159948 2024-12-04T14:55:59,967 DEBUG [RS:0;d214edae7bd2:34477 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46813:46813)] 2024-12-04T14:55:59,980 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T14:55:59,980 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1 2024-12-04T14:55:59,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741833_1009 (size=32) 2024-12-04T14:56:00,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:56:00,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:56:00,402 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:56:00,402 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:00,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:56:00,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T14:56:00,407 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T14:56:00,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:00,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:56:00,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:56:00,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:56:00,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:00,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:56:00,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:56:00,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:56:00,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:00,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:56:00,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T14:56:00,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740 2024-12-04T14:56:00,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740 2024-12-04T14:56:00,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T14:56:00,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T14:56:00,420 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:56:00,422 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T14:56:00,426 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:56:00,427 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66007023, jitterRate=-0.016418710350990295}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:56:00,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733324160394Initializing all the Stores at 1733324160397 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324160397Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324160397Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324160398 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324160398Cleaning up temporary data from old regions at 1733324160419 (+21 ms)Region opened successfully at 1733324160429 (+10 ms) 2024-12-04T14:56:00,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:56:00,429 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T14:56:00,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T14:56:00,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:56:00,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:56:00,430 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T14:56:00,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733324160429Disabling compacts and flushes for region at 1733324160429Disabling writes for close at 1733324160429Writing region close event to WAL at 1733324160430 (+1 ms)Closed at 1733324160430 2024-12-04T14:56:00,434 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:56:00,434 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T14:56:00,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T14:56:00,445 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T14:56:00,448 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T14:56:00,600 DEBUG [d214edae7bd2:45797 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T14:56:00,611 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:00,617 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d214edae7bd2,34477,1733324158096, state=OPENING 2024-12-04T14:56:00,660 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T14:56:00,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:56:00,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:56:00,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:56:00,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:56:00,671 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T14:56:00,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:56:00,846 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T14:56:00,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36825, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T14:56:00,859 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T14:56:00,859 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T14:56:00,860 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T14:56:00,863 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d214edae7bd2%2C34477%2C1733324158096.meta, suffix=.meta, logDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/WALs/d214edae7bd2,34477,1733324158096, archiveDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/oldWALs, maxLogs=32 2024-12-04T14:56:00,875 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/WALs/d214edae7bd2,34477,1733324158096/d214edae7bd2%2C34477%2C1733324158096.meta.1733324160864.meta, exclude list is [], retry=0 2024-12-04T14:56:00,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35153,DS-4a7129ce-3f0f-4ff0-b959-d3d1a01d4347,DISK] 2024-12-04T14:56:00,882 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/WALs/d214edae7bd2,34477,1733324158096/d214edae7bd2%2C34477%2C1733324158096.meta.1733324160864.meta 2024-12-04T14:56:00,882 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46813:46813)] 2024-12-04T14:56:00,882 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:56:00,884 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T14:56:00,887 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T14:56:00,892 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T14:56:00,896 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T14:56:00,896 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:56:00,896 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T14:56:00,897 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T14:56:00,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T14:56:00,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T14:56:00,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:00,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:56:00,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T14:56:00,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T14:56:00,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:00,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:56:00,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T14:56:00,906 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T14:56:00,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:00,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:56:00,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T14:56:00,909 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T14:56:00,909 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:00,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T14:56:00,910 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T14:56:00,911 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740 2024-12-04T14:56:00,914 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740 2024-12-04T14:56:00,916 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T14:56:00,917 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T14:56:00,918 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:56:00,920 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T14:56:00,922 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68374897, jitterRate=0.01886536180973053}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:56:00,922 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T14:56:00,924 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733324160897Writing region info on filesystem at 1733324160897Initializing all the Stores at 1733324160899 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324160899Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324160900 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324160900Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324160900Cleaning up temporary data from old regions at 1733324160917 (+17 ms)Running coprocessor post-open hooks at 1733324160922 (+5 ms)Region opened successfully at 1733324160923 (+1 ms) 2024-12-04T14:56:00,930 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733324160837 2024-12-04T14:56:00,939 DEBUG [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T14:56:00,940 INFO [RS_OPEN_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T14:56:00,941 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:00,944 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d214edae7bd2,34477,1733324158096, state=OPEN 2024-12-04T14:56:00,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:56:00,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T14:56:00,971 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:56:00,971 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T14:56:00,972 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:00,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T14:56:00,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d214edae7bd2,34477,1733324158096 in 300 msec 2024-12-04T14:56:00,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T14:56:00,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 545 msec 2024-12-04T14:56:00,991 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T14:56:00,991 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T14:56:01,009 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:01,011 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:01,031 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:01,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40839, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:01,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5680 sec 2024-12-04T14:56:01,052 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733324161052, completionTime=-1 2024-12-04T14:56:01,055 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T14:56:01,056 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T14:56:01,083 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T14:56:01,083 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733324221083 2024-12-04T14:56:01,083 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733324281083 2024-12-04T14:56:01,083 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-04T14:56:01,086 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,45797,1733324157426-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T14:56:01,086 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,45797,1733324157426-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:56:01,087 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,45797,1733324157426-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:56:01,088 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d214edae7bd2:45797, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:56:01,088 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T14:56:01,089 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T14:56:01,094 DEBUG [master/d214edae7bd2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T14:56:01,116 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.871sec 2024-12-04T14:56:01,117 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T14:56:01,118 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T14:56:01,118 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T14:56:01,119 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T14:56:01,119 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T14:56:01,120 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,45797,1733324157426-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T14:56:01,120 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,45797,1733324157426-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T14:56:01,144 DEBUG [master/d214edae7bd2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T14:56:01,145 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T14:56:01,145 INFO [master/d214edae7bd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d214edae7bd2,45797,1733324157426-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T14:56:01,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:01,197 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T14:56:01,197 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T14:56:01,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:01,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:01,209 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:01,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:01,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:01,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e667ddd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:01,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:01,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:01,218 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:01,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:01,222 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:01,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:01,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:01,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d214edae7bd2,45797,1733324157426 2024-12-04T14:56:01,262 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=9530 2024-12-04T14:56:01,269 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:56:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,45797,1733324157426 2024-12-04T14:56:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b74020b 2024-12-04T14:56:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:56:01,279 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:56:01,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:56:01,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:56:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:01,294 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:56:01,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-04T14:56:01,296 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:01,298 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:56:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:56:01,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741835_1011 (size=959) 2024-12-04T14:56:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:56:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:56:01,713 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1 2024-12-04T14:56:01,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741836_1012 (size=53) 2024-12-04T14:56:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:56:02,127 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:56:02,128 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7618a4771c68a9e87db105604d8452e1, disabling compactions & flushes 2024-12-04T14:56:02,128 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:02,128 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:02,129 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. after waiting 0 ms 2024-12-04T14:56:02,129 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:02,129 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:02,129 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7618a4771c68a9e87db105604d8452e1: Waiting for close lock at 1733324162128Disabling compacts and flushes for region at 1733324162128Disabling writes for close at 1733324162129 (+1 ms)Writing region close event to WAL at 1733324162129Closed at 1733324162129 2024-12-04T14:56:02,134 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:56:02,139 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324162135"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324162135"}]},"ts":"1733324162135"} 2024-12-04T14:56:02,144 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T14:56:02,146 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:56:02,149 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324162146"}]},"ts":"1733324162146"} 2024-12-04T14:56:02,153 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T14:56:02,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7618a4771c68a9e87db105604d8452e1, ASSIGN}] 2024-12-04T14:56:02,159 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7618a4771c68a9e87db105604d8452e1, ASSIGN 2024-12-04T14:56:02,160 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7618a4771c68a9e87db105604d8452e1, ASSIGN; state=OFFLINE, location=d214edae7bd2,34477,1733324158096; forceNewPlan=false, retain=false 2024-12-04T14:56:02,312 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7618a4771c68a9e87db105604d8452e1, regionState=OPENING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:02,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7618a4771c68a9e87db105604d8452e1, ASSIGN because future has completed 2024-12-04T14:56:02,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:56:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:56:02,480 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:02,480 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:56:02,480 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,480 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:56:02,480 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,481 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,483 INFO [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,485 INFO [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7618a4771c68a9e87db105604d8452e1 columnFamilyName A 2024-12-04T14:56:02,485 DEBUG [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:02,486 INFO [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] regionserver.HStore(327): Store=7618a4771c68a9e87db105604d8452e1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:02,487 INFO [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,489 INFO [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7618a4771c68a9e87db105604d8452e1 columnFamilyName B 2024-12-04T14:56:02,489 DEBUG [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:02,489 INFO [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] regionserver.HStore(327): Store=7618a4771c68a9e87db105604d8452e1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:02,490 INFO [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,491 INFO [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7618a4771c68a9e87db105604d8452e1 columnFamilyName C 2024-12-04T14:56:02,492 DEBUG [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:02,492 INFO [StoreOpener-7618a4771c68a9e87db105604d8452e1-1 {}] regionserver.HStore(327): Store=7618a4771c68a9e87db105604d8452e1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:02,493 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,494 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,495 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,496 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,497 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,497 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:56:02,500 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,504 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:56:02,504 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7618a4771c68a9e87db105604d8452e1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60242607, jitterRate=-0.10231520235538483}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:56:02,504 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:02,505 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7618a4771c68a9e87db105604d8452e1: Running coprocessor pre-open hook at 1733324162481Writing region info on filesystem at 1733324162481Initializing all the Stores at 1733324162482 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324162482Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324162483 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324162483Cleaning up temporary data from old regions at 1733324162497 (+14 ms)Running coprocessor post-open hooks at 1733324162505 (+8 ms)Region opened successfully at 1733324162505 2024-12-04T14:56:02,507 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., pid=6, masterSystemTime=1733324162473 2024-12-04T14:56:02,510 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:02,510 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:02,511 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7618a4771c68a9e87db105604d8452e1, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:02,513 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45797 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d214edae7bd2,34477,1733324158096, table=TestAcidGuarantees, region=7618a4771c68a9e87db105604d8452e1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T14:56:02,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:56:02,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T14:56:02,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 in 200 msec 2024-12-04T14:56:02,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T14:56:02,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7618a4771c68a9e87db105604d8452e1, ASSIGN in 366 msec 2024-12-04T14:56:02,528 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:56:02,528 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324162528"}]},"ts":"1733324162528"} 2024-12-04T14:56:02,531 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T14:56:02,533 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:56:02,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2460 sec 2024-12-04T14:56:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T14:56:03,459 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:03,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:03,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:03,471 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f9528a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:03,475 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538d0490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,478 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:03,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:03,486 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:56:03,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,45797,1733324157426 2024-12-04T14:56:03,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38391cd4 2024-12-04T14:56:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:56:03,491 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:56:03,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:03,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:03,498 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e11900a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:03,502 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca0b5f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,504 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:03,504 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:03,505 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387dfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:03,511 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25175df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:03,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:03,514 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:03,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4e6a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:03,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:03,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:03,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,521 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:03,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:03,523 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f338c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:03,527 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10112b3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:03,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:03,530 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:03,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:03,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@723bb0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:03,532 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:03,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@235bf972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:03,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:03,536 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5005e0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:03,539 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:03,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d70ac03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:03,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:03,542 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5e1b26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:03,545 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6acfb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:03,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:03,548 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f9cc25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:03,551 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:03,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189704a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:03,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:03,555 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7299e0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:03,558 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39568e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:03,562 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:03,562 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:03,562 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:03,562 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:03,562 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:03,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:03,564 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:03,565 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:03,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:03,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:03,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:03,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:03,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:03,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:03,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:03,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:03,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:03,570 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:03,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:03,576 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:03,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:03,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-04T14:56:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:03,582 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:03,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:03,583 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:03,585 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:03,588 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:56:03,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:03,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:03,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:03,660 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:03,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:56:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:56:03,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2ffd23fae4eb449290b156e8bdf157d9 is 50, key is test_row_0/A:col10/1733324163653/Put/seqid=0 2024-12-04T14:56:03,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:56:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:03,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:03,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:03,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:03,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:03,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741837_1013 (size=14341) 2024-12-04T14:56:03,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2ffd23fae4eb449290b156e8bdf157d9 2024-12-04T14:56:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:03,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:03,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324223815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:03,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324223820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324223813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:03,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:03,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324223825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:03,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324223823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:56:03,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/89535a6c03bb400899f0f3c3c6ba707a is 50, key is test_row_0/B:col10/1733324163653/Put/seqid=0 2024-12-04T14:56:03,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324223946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324223946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:03,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324223947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:03,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324223946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:03,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324223949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:03,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:56:03,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:03,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:03,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:03,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:03,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:03,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741838_1014 (size=12001) 2024-12-04T14:56:03,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/89535a6c03bb400899f0f3c3c6ba707a 2024-12-04T14:56:04,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/c107dc89858743c5b451ade7443bab01 is 50, key is test_row_0/C:col10/1733324163653/Put/seqid=0 2024-12-04T14:56:04,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741839_1015 (size=12001) 2024-12-04T14:56:04,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/c107dc89858743c5b451ade7443bab01 2024-12-04T14:56:04,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2ffd23fae4eb449290b156e8bdf157d9 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ffd23fae4eb449290b156e8bdf157d9 2024-12-04T14:56:04,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ffd23fae4eb449290b156e8bdf157d9, entries=200, sequenceid=14, filesize=14.0 K 2024-12-04T14:56:04,116 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-04T14:56:04,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:56:04,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/89535a6c03bb400899f0f3c3c6ba707a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/89535a6c03bb400899f0f3c3c6ba707a 2024-12-04T14:56:04,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:04,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:04,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:04,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:04,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:04,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/89535a6c03bb400899f0f3c3c6ba707a, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T14:56:04,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/c107dc89858743c5b451ade7443bab01 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/c107dc89858743c5b451ade7443bab01 2024-12-04T14:56:04,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324224157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324224157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324224157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/c107dc89858743c5b451ade7443bab01, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T14:56:04,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 7618a4771c68a9e87db105604d8452e1 in 510ms, sequenceid=14, compaction requested=false 2024-12-04T14:56:04,178 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T14:56:04,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:04,181 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:04,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-04T14:56:04,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/64934dc1e8c241c9a344295e532a238e is 50, key is test_row_0/A:col10/1733324163818/Put/seqid=0 2024-12-04T14:56:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:56:04,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741840_1016 (size=14341) 2024-12-04T14:56:04,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/64934dc1e8c241c9a344295e532a238e 2024-12-04T14:56:04,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324224249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324224256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:56:04,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:04,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:04,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:04,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/510efa4a780645ab8502cc81e74216d1 is 50, key is test_row_0/B:col10/1733324163818/Put/seqid=0 2024-12-04T14:56:04,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:04,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:04,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741841_1017 (size=12001) 2024-12-04T14:56:04,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/510efa4a780645ab8502cc81e74216d1 2024-12-04T14:56:04,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/8b05c525375f42e2a8d89e70fd3af429 is 50, key is test_row_0/C:col10/1733324163818/Put/seqid=0 2024-12-04T14:56:04,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741842_1018 (size=12001) 2024-12-04T14:56:04,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/8b05c525375f42e2a8d89e70fd3af429 2024-12-04T14:56:04,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/64934dc1e8c241c9a344295e532a238e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/64934dc1e8c241c9a344295e532a238e 2024-12-04T14:56:04,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/64934dc1e8c241c9a344295e532a238e, entries=200, sequenceid=38, filesize=14.0 K 2024-12-04T14:56:04,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324224366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324224367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/510efa4a780645ab8502cc81e74216d1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/510efa4a780645ab8502cc81e74216d1 2024-12-04T14:56:04,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/510efa4a780645ab8502cc81e74216d1, entries=150, sequenceid=38, filesize=11.7 K 2024-12-04T14:56:04,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/8b05c525375f42e2a8d89e70fd3af429 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8b05c525375f42e2a8d89e70fd3af429 2024-12-04T14:56:04,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8b05c525375f42e2a8d89e70fd3af429, entries=150, sequenceid=38, filesize=11.7 K 2024-12-04T14:56:04,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 7618a4771c68a9e87db105604d8452e1 in 245ms, sequenceid=38, compaction requested=false 2024-12-04T14:56:04,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:04,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T14:56:04,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:04,435 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-04T14:56:04,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/7007a544dfec4d81937dc4b1a9466ae5 is 50, key is test_row_1/A:col10/1733324164250/Put/seqid=0 2024-12-04T14:56:04,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741843_1019 (size=9657) 2024-12-04T14:56:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:04,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324224586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324224593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324224600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324224601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324224596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324224718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:56:04,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324224719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324224719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324224719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,871 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/7007a544dfec4d81937dc4b1a9466ae5 2024-12-04T14:56:04,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/b88bbcfc185c4008a079309c4bd77e10 is 50, key is test_row_1/B:col10/1733324164250/Put/seqid=0 2024-12-04T14:56:04,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324224905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741844_1020 (size=9657) 2024-12-04T14:56:04,924 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/b88bbcfc185c4008a079309c4bd77e10 2024-12-04T14:56:04,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324224937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324224937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/1e814308ba1c4529b48a14c167c01b7f is 50, key is test_row_1/C:col10/1733324164250/Put/seqid=0 2024-12-04T14:56:04,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324224946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324224946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:04,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741845_1021 (size=9657) 2024-12-04T14:56:04,966 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/1e814308ba1c4529b48a14c167c01b7f 2024-12-04T14:56:04,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/7007a544dfec4d81937dc4b1a9466ae5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7007a544dfec4d81937dc4b1a9466ae5 2024-12-04T14:56:05,005 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7007a544dfec4d81937dc4b1a9466ae5, entries=100, sequenceid=50, filesize=9.4 K 2024-12-04T14:56:05,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/b88bbcfc185c4008a079309c4bd77e10 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b88bbcfc185c4008a079309c4bd77e10 2024-12-04T14:56:05,027 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b88bbcfc185c4008a079309c4bd77e10, entries=100, sequenceid=50, filesize=9.4 K 2024-12-04T14:56:05,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/1e814308ba1c4529b48a14c167c01b7f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/1e814308ba1c4529b48a14c167c01b7f 2024-12-04T14:56:05,054 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/1e814308ba1c4529b48a14c167c01b7f, entries=100, sequenceid=50, filesize=9.4 K 2024-12-04T14:56:05,056 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 7618a4771c68a9e87db105604d8452e1 in 622ms, sequenceid=50, compaction requested=true 2024-12-04T14:56:05,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:05,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:05,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-04T14:56:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-04T14:56:05,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T14:56:05,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4700 sec 2024-12-04T14:56:05,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.5000 sec 2024-12-04T14:56:05,255 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:05,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-04T14:56:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:05,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/79b20b2b66684f68aa972543fdb2b273 is 50, key is test_row_0/A:col10/1733324165252/Put/seqid=0 2024-12-04T14:56:05,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324225268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324225272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324225275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741846_1022 (size=12001) 2024-12-04T14:56:05,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/79b20b2b66684f68aa972543fdb2b273 2024-12-04T14:56:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324225275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/c69ff94f4c8d4068b33bf4221464339e is 50, key is test_row_0/B:col10/1733324165252/Put/seqid=0 2024-12-04T14:56:05,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741847_1023 (size=12001) 2024-12-04T14:56:05,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324225386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324225386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324225396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324225395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324225415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324225595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324225595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324225607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324225608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:05,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T14:56:05,736 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:05,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-04T14:56:05,742 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:56:05,745 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:05,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/c69ff94f4c8d4068b33bf4221464339e 2024-12-04T14:56:05,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:05,749 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T14:56:05,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/8ef411078ad840a5aaf6196dd9ec8d12 is 50, key is test_row_0/C:col10/1733324165252/Put/seqid=0 2024-12-04T14:56:05,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741848_1024 (size=12001) 2024-12-04T14:56:05,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/8ef411078ad840a5aaf6196dd9ec8d12 2024-12-04T14:56:05,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/79b20b2b66684f68aa972543fdb2b273 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/79b20b2b66684f68aa972543fdb2b273 2024-12-04T14:56:05,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/79b20b2b66684f68aa972543fdb2b273, entries=150, sequenceid=77, filesize=11.7 K 2024-12-04T14:56:05,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/c69ff94f4c8d4068b33bf4221464339e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c69ff94f4c8d4068b33bf4221464339e 2024-12-04T14:56:05,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c69ff94f4c8d4068b33bf4221464339e, entries=150, sequenceid=77, filesize=11.7 K 2024-12-04T14:56:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:56:05,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/8ef411078ad840a5aaf6196dd9ec8d12 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8ef411078ad840a5aaf6196dd9ec8d12 2024-12-04T14:56:05,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8ef411078ad840a5aaf6196dd9ec8d12, entries=150, sequenceid=77, filesize=11.7 K 2024-12-04T14:56:05,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 7618a4771c68a9e87db105604d8452e1 in 639ms, sequenceid=77, compaction requested=true 2024-12-04T14:56:05,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:05,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T14:56:05,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:05,902 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-04T14:56:05,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:05,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:05,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:05,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:05,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:05,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:05,913 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:05,913 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:05,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/f3dbe4e666fd424e9d2da4bb0c632bb8 is 50, key is test_row_1/A:col10/1733324165271/Put/seqid=0 2024-12-04T14:56:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:05,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:05,918 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:05,918 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:05,920 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/B is initiating minor compaction (all files) 2024-12-04T14:56:05,920 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/B in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:05,921 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/89535a6c03bb400899f0f3c3c6ba707a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/510efa4a780645ab8502cc81e74216d1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b88bbcfc185c4008a079309c4bd77e10, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c69ff94f4c8d4068b33bf4221464339e] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=44.6 K 2024-12-04T14:56:05,922 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/A is initiating minor compaction (all files) 2024-12-04T14:56:05,922 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/A in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:05,922 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ffd23fae4eb449290b156e8bdf157d9, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/64934dc1e8c241c9a344295e532a238e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7007a544dfec4d81937dc4b1a9466ae5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/79b20b2b66684f68aa972543fdb2b273] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=49.2 K 2024-12-04T14:56:05,925 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 89535a6c03bb400899f0f3c3c6ba707a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733324163650 2024-12-04T14:56:05,926 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 510efa4a780645ab8502cc81e74216d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733324163815 2024-12-04T14:56:05,928 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b88bbcfc185c4008a079309c4bd77e10, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733324164217 2024-12-04T14:56:05,928 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ffd23fae4eb449290b156e8bdf157d9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733324163609 2024-12-04T14:56:05,929 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c69ff94f4c8d4068b33bf4221464339e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733324164591 2024-12-04T14:56:05,929 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64934dc1e8c241c9a344295e532a238e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733324163806 2024-12-04T14:56:05,930 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7007a544dfec4d81937dc4b1a9466ae5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733324164217 2024-12-04T14:56:05,931 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b20b2b66684f68aa972543fdb2b273, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733324164591 2024-12-04T14:56:05,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741849_1025 (size=9657) 2024-12-04T14:56:05,951 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/f3dbe4e666fd424e9d2da4bb0c632bb8 2024-12-04T14:56:05,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/baa5d4a467ac43bea31ee056543c9c3a is 50, key is test_row_1/B:col10/1733324165271/Put/seqid=0 2024-12-04T14:56:05,985 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#A#compaction#14 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:05,987 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/a879a334b4b14c76aa3013161fdb519d is 50, key is test_row_0/A:col10/1733324165252/Put/seqid=0 2024-12-04T14:56:05,990 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#B#compaction#13 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:05,991 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/663a359d3adf4ef29c7b7e7aeda7f60b is 50, key is test_row_0/B:col10/1733324165252/Put/seqid=0 2024-12-04T14:56:06,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741851_1027 (size=12139) 2024-12-04T14:56:06,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741850_1026 (size=9657) 2024-12-04T14:56:06,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741852_1028 (size=12139) 2024-12-04T14:56:06,055 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/663a359d3adf4ef29c7b7e7aeda7f60b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/663a359d3adf4ef29c7b7e7aeda7f60b 2024-12-04T14:56:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:56:06,078 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/a879a334b4b14c76aa3013161fdb519d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/a879a334b4b14c76aa3013161fdb519d 2024-12-04T14:56:06,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324226066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324226071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324226077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324226080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,099 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/A of 7618a4771c68a9e87db105604d8452e1 into a879a334b4b14c76aa3013161fdb519d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:06,099 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:06,101 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/B of 7618a4771c68a9e87db105604d8452e1 into 663a359d3adf4ef29c7b7e7aeda7f60b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:06,101 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:06,102 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/A, priority=12, startTime=1733324165906; duration=0sec 2024-12-04T14:56:06,103 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:06,103 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:A 2024-12-04T14:56:06,103 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:06,106 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:06,106 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/C is initiating minor compaction (all files) 2024-12-04T14:56:06,106 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/B, priority=12, startTime=1733324165909; duration=0sec 2024-12-04T14:56:06,106 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/C in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:06,106 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:06,106 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/c107dc89858743c5b451ade7443bab01, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8b05c525375f42e2a8d89e70fd3af429, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/1e814308ba1c4529b48a14c167c01b7f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8ef411078ad840a5aaf6196dd9ec8d12] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=44.6 K 2024-12-04T14:56:06,106 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:B 2024-12-04T14:56:06,108 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting c107dc89858743c5b451ade7443bab01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733324163650 2024-12-04T14:56:06,109 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b05c525375f42e2a8d89e70fd3af429, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733324163815 2024-12-04T14:56:06,111 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e814308ba1c4529b48a14c167c01b7f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733324164217 2024-12-04T14:56:06,113 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef411078ad840a5aaf6196dd9ec8d12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733324164591 2024-12-04T14:56:06,152 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#C#compaction#16 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:06,153 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/bddc4df34e39423c801199a86fdcfbb1 is 50, key is test_row_0/C:col10/1733324165252/Put/seqid=0 2024-12-04T14:56:06,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741853_1029 (size=12139) 2024-12-04T14:56:06,182 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/bddc4df34e39423c801199a86fdcfbb1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bddc4df34e39423c801199a86fdcfbb1 2024-12-04T14:56:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324226185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,197 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/C of 7618a4771c68a9e87db105604d8452e1 into bddc4df34e39423c801199a86fdcfbb1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:06,197 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:06,197 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/C, priority=12, startTime=1733324165910; duration=0sec 2024-12-04T14:56:06,197 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:06,197 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:C 2024-12-04T14:56:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324226196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324226196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324226196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:56:06,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324226405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,409 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:56:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324226406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324226406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324226406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324226436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,443 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/baa5d4a467ac43bea31ee056543c9c3a 2024-12-04T14:56:06,445 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_2' in 7618a4771c68a9e87db105604d8452e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:56:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/bde0990eb8714e2fb6a23291b6b22372 is 50, key is test_row_1/C:col10/1733324165271/Put/seqid=0 2024-12-04T14:56:06,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741854_1030 (size=9657) 2024-12-04T14:56:06,523 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/bde0990eb8714e2fb6a23291b6b22372 2024-12-04T14:56:06,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/f3dbe4e666fd424e9d2da4bb0c632bb8 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f3dbe4e666fd424e9d2da4bb0c632bb8 2024-12-04T14:56:06,558 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f3dbe4e666fd424e9d2da4bb0c632bb8, entries=100, sequenceid=86, filesize=9.4 K 2024-12-04T14:56:06,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/baa5d4a467ac43bea31ee056543c9c3a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/baa5d4a467ac43bea31ee056543c9c3a 2024-12-04T14:56:06,579 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/baa5d4a467ac43bea31ee056543c9c3a, entries=100, sequenceid=86, filesize=9.4 K 2024-12-04T14:56:06,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/bde0990eb8714e2fb6a23291b6b22372 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bde0990eb8714e2fb6a23291b6b22372 2024-12-04T14:56:06,597 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bde0990eb8714e2fb6a23291b6b22372, entries=100, sequenceid=86, filesize=9.4 K 2024-12-04T14:56:06,601 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 7618a4771c68a9e87db105604d8452e1 in 698ms, sequenceid=86, compaction requested=false 2024-12-04T14:56:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-04T14:56:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-04T14:56:06,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T14:56:06,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 860 msec 2024-12-04T14:56:06,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 874 msec 2024-12-04T14:56:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:06,719 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:06,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-12-04T14:56:06,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/f1bd5c7a964b4a409fdf06baf38e05b6 is 50, key is test_row_0/A:col10/1733324166057/Put/seqid=0 2024-12-04T14:56:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324226725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324226727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324226729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324226736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741855_1031 (size=12001) 2024-12-04T14:56:06,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/f1bd5c7a964b4a409fdf06baf38e05b6 2024-12-04T14:56:06,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/2710df85b43f460e8bff95a61a984614 is 50, key is test_row_0/B:col10/1733324166057/Put/seqid=0 2024-12-04T14:56:06,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741856_1032 (size=12001) 2024-12-04T14:56:06,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/2710df85b43f460e8bff95a61a984614 2024-12-04T14:56:06,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/5734a2fc372b4641a89ddd406529c21c is 50, key is test_row_0/C:col10/1733324166057/Put/seqid=0 2024-12-04T14:56:06,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324226845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324226845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324226845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:06,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741857_1033 (size=12001) 2024-12-04T14:56:06,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/5734a2fc372b4641a89ddd406529c21c 2024-12-04T14:56:06,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/f1bd5c7a964b4a409fdf06baf38e05b6 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f1bd5c7a964b4a409fdf06baf38e05b6 2024-12-04T14:56:06,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T14:56:06,886 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:06,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-04T14:56:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:56:06,892 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:06,894 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:06,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:06,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f1bd5c7a964b4a409fdf06baf38e05b6, entries=150, sequenceid=119, filesize=11.7 K 2024-12-04T14:56:06,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/2710df85b43f460e8bff95a61a984614 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2710df85b43f460e8bff95a61a984614 2024-12-04T14:56:06,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2710df85b43f460e8bff95a61a984614, entries=150, sequenceid=119, filesize=11.7 K 2024-12-04T14:56:06,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/5734a2fc372b4641a89ddd406529c21c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/5734a2fc372b4641a89ddd406529c21c 2024-12-04T14:56:06,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/5734a2fc372b4641a89ddd406529c21c, entries=150, sequenceid=119, filesize=11.7 K 2024-12-04T14:56:06,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 7618a4771c68a9e87db105604d8452e1 in 216ms, sequenceid=119, compaction requested=true 2024-12-04T14:56:06,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:06,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:06,936 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:06,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:06,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:06,937 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:06,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:06,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:06,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:06,938 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:06,939 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/A is initiating minor compaction (all files) 2024-12-04T14:56:06,939 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/A in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:06,939 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/a879a334b4b14c76aa3013161fdb519d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f3dbe4e666fd424e9d2da4bb0c632bb8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f1bd5c7a964b4a409fdf06baf38e05b6] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=33.0 K 2024-12-04T14:56:06,939 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:06,939 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/B is initiating minor compaction (all files) 2024-12-04T14:56:06,940 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/B in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:06,940 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/663a359d3adf4ef29c7b7e7aeda7f60b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/baa5d4a467ac43bea31ee056543c9c3a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2710df85b43f460e8bff95a61a984614] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=33.0 K 2024-12-04T14:56:06,940 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting a879a334b4b14c76aa3013161fdb519d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733324164591 2024-12-04T14:56:06,942 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3dbe4e666fd424e9d2da4bb0c632bb8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733324165269 2024-12-04T14:56:06,942 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 663a359d3adf4ef29c7b7e7aeda7f60b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733324164591 2024-12-04T14:56:06,945 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1bd5c7a964b4a409fdf06baf38e05b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324166057 2024-12-04T14:56:06,945 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting baa5d4a467ac43bea31ee056543c9c3a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733324165269 2024-12-04T14:56:06,948 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 2710df85b43f460e8bff95a61a984614, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324166057 2024-12-04T14:56:06,989 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#A#compaction#21 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:06,989 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#B#compaction#22 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:06,990 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/ee318cbb4fe74b179fa51e1321014913 is 50, key is test_row_0/A:col10/1733324166057/Put/seqid=0 2024-12-04T14:56:06,990 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/9467ef5401134f1c8b21fa54068a893b is 50, key is test_row_0/B:col10/1733324166057/Put/seqid=0 2024-12-04T14:56:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:56:07,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741858_1034 (size=12241) 2024-12-04T14:56:07,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741859_1035 (size=12241) 2024-12-04T14:56:07,035 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/ee318cbb4fe74b179fa51e1321014913 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/ee318cbb4fe74b179fa51e1321014913 2024-12-04T14:56:07,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T14:56:07,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:07,050 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-12-04T14:56:07,052 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/A of 7618a4771c68a9e87db105604d8452e1 into ee318cbb4fe74b179fa51e1321014913(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:07,052 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:07,052 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/A, priority=13, startTime=1733324166936; duration=0sec 2024-12-04T14:56:07,053 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:07,053 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:A 2024-12-04T14:56:07,053 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:07,061 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:07,061 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/C is initiating minor compaction (all files) 2024-12-04T14:56:07,061 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/C in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:07,061 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bddc4df34e39423c801199a86fdcfbb1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bde0990eb8714e2fb6a23291b6b22372, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/5734a2fc372b4641a89ddd406529c21c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=33.0 K 2024-12-04T14:56:07,062 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting bddc4df34e39423c801199a86fdcfbb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733324164591 2024-12-04T14:56:07,063 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting bde0990eb8714e2fb6a23291b6b22372, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733324165269 2024-12-04T14:56:07,064 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5734a2fc372b4641a89ddd406529c21c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324166057 2024-12-04T14:56:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/8edb7c3364b54c7ca992ff854bb56f82 is 50, key is test_row_0/A:col10/1733324166728/Put/seqid=0 2024-12-04T14:56:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:07,096 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#C#compaction#24 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:07,097 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/ad48e631386f4939a29d39c42b83d70d is 50, key is test_row_0/C:col10/1733324166057/Put/seqid=0 2024-12-04T14:56:07,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741860_1036 (size=12001) 2024-12-04T14:56:07,110 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/8edb7c3364b54c7ca992ff854bb56f82 2024-12-04T14:56:07,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/3671a6a3d51c42d2a9af00caf4e24e69 is 50, key is test_row_0/B:col10/1733324166728/Put/seqid=0 2024-12-04T14:56:07,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741861_1037 (size=12241) 2024-12-04T14:56:07,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741862_1038 (size=12001) 2024-12-04T14:56:07,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324227169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324227176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324227172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:56:07,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324227245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324227285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324227285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324227285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,433 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/9467ef5401134f1c8b21fa54068a893b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/9467ef5401134f1c8b21fa54068a893b 2024-12-04T14:56:07,450 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/B of 7618a4771c68a9e87db105604d8452e1 into 9467ef5401134f1c8b21fa54068a893b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:07,450 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:07,450 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/B, priority=13, startTime=1733324166937; duration=0sec 2024-12-04T14:56:07,451 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:07,451 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:B 2024-12-04T14:56:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324227495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324227496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324227505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:56:07,564 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/ad48e631386f4939a29d39c42b83d70d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/ad48e631386f4939a29d39c42b83d70d 2024-12-04T14:56:07,573 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/3671a6a3d51c42d2a9af00caf4e24e69 2024-12-04T14:56:07,580 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/C of 7618a4771c68a9e87db105604d8452e1 into ad48e631386f4939a29d39c42b83d70d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:07,581 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:07,581 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/C, priority=13, startTime=1733324166937; duration=0sec 2024-12-04T14:56:07,581 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:07,581 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:C 2024-12-04T14:56:07,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/7750bdf426884f03abc3588bfdce1a3b is 50, key is test_row_0/C:col10/1733324166728/Put/seqid=0 2024-12-04T14:56:07,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741863_1039 (size=12001) 2024-12-04T14:56:07,750 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T14:56:07,751 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T14:56:07,754 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T14:56:07,754 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T14:56:07,754 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T14:56:07,754 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T14:56:07,756 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-04T14:56:07,756 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-04T14:56:07,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324227805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324227805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:07,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324227815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:07,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:07,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:08,003 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/7750bdf426884f03abc3588bfdce1a3b 2024-12-04T14:56:08,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/8edb7c3364b54c7ca992ff854bb56f82 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/8edb7c3364b54c7ca992ff854bb56f82 2024-12-04T14:56:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:56:08,026 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/8edb7c3364b54c7ca992ff854bb56f82, entries=150, sequenceid=126, filesize=11.7 K 2024-12-04T14:56:08,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/3671a6a3d51c42d2a9af00caf4e24e69 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3671a6a3d51c42d2a9af00caf4e24e69 2024-12-04T14:56:08,041 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3671a6a3d51c42d2a9af00caf4e24e69, entries=150, sequenceid=126, filesize=11.7 K 2024-12-04T14:56:08,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/7750bdf426884f03abc3588bfdce1a3b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/7750bdf426884f03abc3588bfdce1a3b 2024-12-04T14:56:08,059 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/7750bdf426884f03abc3588bfdce1a3b, entries=150, sequenceid=126, filesize=11.7 K 2024-12-04T14:56:08,061 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 7618a4771c68a9e87db105604d8452e1 in 1011ms, sequenceid=126, compaction requested=false 2024-12-04T14:56:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-04T14:56:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-04T14:56:08,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T14:56:08,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1700 sec 2024-12-04T14:56:08,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1870 sec 2024-12-04T14:56:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:08,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-12-04T14:56:08,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/96e056fa022944f4a3f75f69deefe458 is 50, key is test_row_0/A:col10/1733324167168/Put/seqid=0 2024-12-04T14:56:08,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324228269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:08,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741864_1040 (size=12151) 2024-12-04T14:56:08,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/96e056fa022944f4a3f75f69deefe458 2024-12-04T14:56:08,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:08,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324228315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324228315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:08,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324228325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:08,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/04b5eb7bdc8547d6aeb8b4cc428b33b5 is 50, key is test_row_0/B:col10/1733324167168/Put/seqid=0 2024-12-04T14:56:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:08,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741865_1041 (size=12151) 2024-12-04T14:56:08,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324228375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:08,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324228455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:08,460 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_2' in 7618a4771c68a9e87db105604d8452e1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:56:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:08,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324228585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:08,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/04b5eb7bdc8547d6aeb8b4cc428b33b5 2024-12-04T14:56:08,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/f379dbda303a481f9be805ff9cc787c1 is 50, key is test_row_0/C:col10/1733324167168/Put/seqid=0 2024-12-04T14:56:08,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741866_1042 (size=12151) 2024-12-04T14:56:08,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324228895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T14:56:09,035 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:09,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-04T14:56:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:56:09,039 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:09,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:09,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:56:09,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/f379dbda303a481f9be805ff9cc787c1 2024-12-04T14:56:09,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/96e056fa022944f4a3f75f69deefe458 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/96e056fa022944f4a3f75f69deefe458 2024-12-04T14:56:09,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T14:56:09,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:09,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/96e056fa022944f4a3f75f69deefe458, entries=150, sequenceid=159, filesize=11.9 K 2024-12-04T14:56:09,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/04b5eb7bdc8547d6aeb8b4cc428b33b5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/04b5eb7bdc8547d6aeb8b4cc428b33b5 2024-12-04T14:56:09,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/04b5eb7bdc8547d6aeb8b4cc428b33b5, entries=150, sequenceid=159, filesize=11.9 K 2024-12-04T14:56:09,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/f379dbda303a481f9be805ff9cc787c1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/f379dbda303a481f9be805ff9cc787c1 2024-12-04T14:56:09,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/f379dbda303a481f9be805ff9cc787c1, entries=150, sequenceid=159, filesize=11.9 K 2024-12-04T14:56:09,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 7618a4771c68a9e87db105604d8452e1 in 999ms, sequenceid=159, compaction requested=true 2024-12-04T14:56:09,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:09,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:09,259 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:09,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:09,259 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:09,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:09,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:09,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:09,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:09,264 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:09,264 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:09,264 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/A is initiating minor compaction (all files) 2024-12-04T14:56:09,264 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/B is initiating minor compaction (all files) 2024-12-04T14:56:09,264 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/A in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,265 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/ee318cbb4fe74b179fa51e1321014913, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/8edb7c3364b54c7ca992ff854bb56f82, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/96e056fa022944f4a3f75f69deefe458] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=35.5 K 2024-12-04T14:56:09,265 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/B in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,265 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/9467ef5401134f1c8b21fa54068a893b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3671a6a3d51c42d2a9af00caf4e24e69, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/04b5eb7bdc8547d6aeb8b4cc428b33b5] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=35.5 K 2024-12-04T14:56:09,265 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee318cbb4fe74b179fa51e1321014913, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324166057 2024-12-04T14:56:09,266 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 9467ef5401134f1c8b21fa54068a893b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324166057 2024-12-04T14:56:09,266 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 3671a6a3d51c42d2a9af00caf4e24e69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733324166722 2024-12-04T14:56:09,267 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8edb7c3364b54c7ca992ff854bb56f82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733324166722 2024-12-04T14:56:09,267 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b5eb7bdc8547d6aeb8b4cc428b33b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733324167152 2024-12-04T14:56:09,268 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96e056fa022944f4a3f75f69deefe458, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733324167152 2024-12-04T14:56:09,285 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:09,286 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/4a6b1adc492f4099992070b237ac7609 is 50, key is test_row_0/B:col10/1733324167168/Put/seqid=0 2024-12-04T14:56:09,291 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#A#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:09,293 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/eaa33088130740d399a8f3ce247790fc is 50, key is test_row_0/A:col10/1733324167168/Put/seqid=0 2024-12-04T14:56:09,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741867_1043 (size=12493) 2024-12-04T14:56:09,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741868_1044 (size=12493) 2024-12-04T14:56:09,324 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/eaa33088130740d399a8f3ce247790fc as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/eaa33088130740d399a8f3ce247790fc 2024-12-04T14:56:09,342 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/A of 7618a4771c68a9e87db105604d8452e1 into eaa33088130740d399a8f3ce247790fc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:09,343 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:09,343 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/A, priority=13, startTime=1733324169259; duration=0sec 2024-12-04T14:56:09,343 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:09,343 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:A 2024-12-04T14:56:09,347 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:09,349 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:09,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-04T14:56:09,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T14:56:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,352 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:09,352 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/C is initiating minor compaction (all files) 2024-12-04T14:56:09,353 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/C in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,353 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/ad48e631386f4939a29d39c42b83d70d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/7750bdf426884f03abc3588bfdce1a3b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/f379dbda303a481f9be805ff9cc787c1] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=35.5 K 2024-12-04T14:56:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:56:09,358 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad48e631386f4939a29d39c42b83d70d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733324166057 2024-12-04T14:56:09,359 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7750bdf426884f03abc3588bfdce1a3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733324166722 2024-12-04T14:56:09,360 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting f379dbda303a481f9be805ff9cc787c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733324167152 2024-12-04T14:56:09,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2a0b6a81e382462c9eab44a85422e2d2 is 50, key is test_row_0/A:col10/1733324169344/Put/seqid=0 2024-12-04T14:56:09,391 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#C#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:09,392 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/2a0e0755bfe44cf7932f35701e457cb1 is 50, key is test_row_0/C:col10/1733324167168/Put/seqid=0 2024-12-04T14:56:09,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741869_1045 (size=16931) 2024-12-04T14:56:09,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324229413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324229413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324229415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324229415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741870_1046 (size=12493) 2024-12-04T14:56:09,450 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/2a0e0755bfe44cf7932f35701e457cb1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2a0e0755bfe44cf7932f35701e457cb1 2024-12-04T14:56:09,467 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/C of 7618a4771c68a9e87db105604d8452e1 into 2a0e0755bfe44cf7932f35701e457cb1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:09,467 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:09,468 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/C, priority=13, startTime=1733324169260; duration=0sec 2024-12-04T14:56:09,468 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:09,468 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:C 2024-12-04T14:56:09,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T14:56:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324229526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324229526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324229527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324229527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T14:56:09,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:09,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:56:09,717 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/4a6b1adc492f4099992070b237ac7609 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/4a6b1adc492f4099992070b237ac7609 2024-12-04T14:56:09,728 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/B of 7618a4771c68a9e87db105604d8452e1 into 4a6b1adc492f4099992070b237ac7609(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:09,728 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:09,728 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/B, priority=13, startTime=1733324169259; duration=0sec 2024-12-04T14:56:09,729 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:09,729 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:B 2024-12-04T14:56:09,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324229736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324229736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324229736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324229736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:09,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2a0b6a81e382462c9eab44a85422e2d2 2024-12-04T14:56:09,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T14:56:09,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/97d0f6baa87d4371bb86478af0feccca is 50, key is test_row_0/B:col10/1733324169344/Put/seqid=0 2024-12-04T14:56:09,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741871_1047 (size=12151) 2024-12-04T14:56:09,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/97d0f6baa87d4371bb86478af0feccca 2024-12-04T14:56:09,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/29850f880b264ab5b8bdbab6fc5b30aa is 50, key is test_row_0/C:col10/1733324169344/Put/seqid=0 2024-12-04T14:56:09,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741872_1048 (size=12151) 2024-12-04T14:56:09,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T14:56:09,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:09,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:09,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:10,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324230045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324230045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324230045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324230045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T14:56:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:10,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:10,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:56:10,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/29850f880b264ab5b8bdbab6fc5b30aa 2024-12-04T14:56:10,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T14:56:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:10,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:10,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:10,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2a0b6a81e382462c9eab44a85422e2d2 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2a0b6a81e382462c9eab44a85422e2d2 2024-12-04T14:56:10,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2a0b6a81e382462c9eab44a85422e2d2, entries=250, sequenceid=172, filesize=16.5 K 2024-12-04T14:56:10,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/97d0f6baa87d4371bb86478af0feccca as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/97d0f6baa87d4371bb86478af0feccca 2024-12-04T14:56:10,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/97d0f6baa87d4371bb86478af0feccca, entries=150, sequenceid=172, filesize=11.9 K 2024-12-04T14:56:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/29850f880b264ab5b8bdbab6fc5b30aa as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/29850f880b264ab5b8bdbab6fc5b30aa 2024-12-04T14:56:10,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/29850f880b264ab5b8bdbab6fc5b30aa, entries=150, sequenceid=172, filesize=11.9 K 2024-12-04T14:56:10,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 7618a4771c68a9e87db105604d8452e1 in 958ms, sequenceid=172, compaction requested=false 2024-12-04T14:56:10,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:10,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T14:56:10,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:10,432 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:56:10,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/e3fb3e0fe5d04f528af7bf87d3519804 is 50, key is test_row_0/A:col10/1733324169412/Put/seqid=0 2024-12-04T14:56:10,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741873_1049 (size=12151) 2024-12-04T14:56:10,454 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/e3fb3e0fe5d04f528af7bf87d3519804 2024-12-04T14:56:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/f9c1159236fe4d8cb1dda5f5e5a4535c is 50, key is test_row_0/B:col10/1733324169412/Put/seqid=0 2024-12-04T14:56:10,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741874_1050 (size=12151) 2024-12-04T14:56:10,488 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/f9c1159236fe4d8cb1dda5f5e5a4535c 2024-12-04T14:56:10,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/a672f41a261f45e9aae22f943e6f0fc1 is 50, key is test_row_0/C:col10/1733324169412/Put/seqid=0 2024-12-04T14:56:10,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741875_1051 (size=12151) 2024-12-04T14:56:10,517 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/a672f41a261f45e9aae22f943e6f0fc1 2024-12-04T14:56:10,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/e3fb3e0fe5d04f528af7bf87d3519804 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e3fb3e0fe5d04f528af7bf87d3519804 2024-12-04T14:56:10,539 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e3fb3e0fe5d04f528af7bf87d3519804, entries=150, sequenceid=199, filesize=11.9 K 2024-12-04T14:56:10,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/f9c1159236fe4d8cb1dda5f5e5a4535c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/f9c1159236fe4d8cb1dda5f5e5a4535c 2024-12-04T14:56:10,552 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/f9c1159236fe4d8cb1dda5f5e5a4535c, entries=150, sequenceid=199, filesize=11.9 K 2024-12-04T14:56:10,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/a672f41a261f45e9aae22f943e6f0fc1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a672f41a261f45e9aae22f943e6f0fc1 2024-12-04T14:56:10,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:10,571 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a672f41a261f45e9aae22f943e6f0fc1, entries=150, sequenceid=199, filesize=11.9 K 2024-12-04T14:56:10,575 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7618a4771c68a9e87db105604d8452e1 in 142ms, sequenceid=199, compaction requested=true 2024-12-04T14:56:10,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:10,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-04T14:56:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:10,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-04T14:56:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-04T14:56:10,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T14:56:10,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5420 sec 2024-12-04T14:56:10,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/7570e63617cf4dafbb587ec761c6d684 is 50, key is test_row_0/A:col10/1733324170573/Put/seqid=0 2024-12-04T14:56:10,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5510 sec 2024-12-04T14:56:10,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741876_1052 (size=14541) 2024-12-04T14:56:10,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/7570e63617cf4dafbb587ec761c6d684 2024-12-04T14:56:10,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/3ec72e3a6ccc4008b33fc2820a660cb5 is 50, key is test_row_0/B:col10/1733324170573/Put/seqid=0 2024-12-04T14:56:10,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324230628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324230630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324230630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324230633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741877_1053 (size=12151) 2024-12-04T14:56:10,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324230745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324230745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324230755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324230755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324230955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324230955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324230966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324230966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/3ec72e3a6ccc4008b33fc2820a660cb5 2024-12-04T14:56:11,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/65ddc623977947dcbb4ddbd4e0ef79fb is 50, key is test_row_0/C:col10/1733324170573/Put/seqid=0 2024-12-04T14:56:11,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741878_1054 (size=12151) 2024-12-04T14:56:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T14:56:11,186 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:11,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-04T14:56:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:56:11,190 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:11,195 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:11,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:11,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324231265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324231265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324231284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324231285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:56:11,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:56:11,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:11,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:11,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:11,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:11,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:56:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:11,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:56:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:11,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/65ddc623977947dcbb4ddbd4e0ef79fb 2024-12-04T14:56:11,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/7570e63617cf4dafbb587ec761c6d684 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7570e63617cf4dafbb587ec761c6d684 2024-12-04T14:56:11,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7570e63617cf4dafbb587ec761c6d684, entries=200, sequenceid=212, filesize=14.2 K 2024-12-04T14:56:11,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/3ec72e3a6ccc4008b33fc2820a660cb5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3ec72e3a6ccc4008b33fc2820a660cb5 2024-12-04T14:56:11,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3ec72e3a6ccc4008b33fc2820a660cb5, entries=150, sequenceid=212, filesize=11.9 K 2024-12-04T14:56:11,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/65ddc623977947dcbb4ddbd4e0ef79fb as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/65ddc623977947dcbb4ddbd4e0ef79fb 2024-12-04T14:56:11,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/65ddc623977947dcbb4ddbd4e0ef79fb, entries=150, sequenceid=212, filesize=11.9 K 2024-12-04T14:56:11,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 7618a4771c68a9e87db105604d8452e1 in 996ms, sequenceid=212, compaction requested=true 2024-12-04T14:56:11,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:11,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:11,573 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:11,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:11,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:11,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:11,574 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:11,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:11,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:11,576 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:11,576 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/A is initiating minor compaction (all files) 2024-12-04T14:56:11,576 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/A in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:11,577 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/eaa33088130740d399a8f3ce247790fc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2a0b6a81e382462c9eab44a85422e2d2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e3fb3e0fe5d04f528af7bf87d3519804, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7570e63617cf4dafbb587ec761c6d684] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=54.8 K 2024-12-04T14:56:11,577 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:11,577 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaa33088130740d399a8f3ce247790fc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733324167152 2024-12-04T14:56:11,577 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/B is initiating minor compaction (all files) 2024-12-04T14:56:11,578 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/B in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:11,578 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/4a6b1adc492f4099992070b237ac7609, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/97d0f6baa87d4371bb86478af0feccca, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/f9c1159236fe4d8cb1dda5f5e5a4535c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3ec72e3a6ccc4008b33fc2820a660cb5] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=47.8 K 2024-12-04T14:56:11,578 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a0b6a81e382462c9eab44a85422e2d2, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733324169329 2024-12-04T14:56:11,578 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a6b1adc492f4099992070b237ac7609, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733324167152 2024-12-04T14:56:11,579 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 97d0f6baa87d4371bb86478af0feccca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733324169333 2024-12-04T14:56:11,579 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3fb3e0fe5d04f528af7bf87d3519804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733324169405 2024-12-04T14:56:11,580 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting f9c1159236fe4d8cb1dda5f5e5a4535c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733324169405 2024-12-04T14:56:11,580 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7570e63617cf4dafbb587ec761c6d684, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324170570 2024-12-04T14:56:11,581 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec72e3a6ccc4008b33fc2820a660cb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324170570 2024-12-04T14:56:11,610 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:11,611 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2fd86e2ebdd244d6aada49137fa08274 is 50, key is test_row_0/A:col10/1733324170573/Put/seqid=0 2024-12-04T14:56:11,613 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:11,614 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/bb866eff2720498fbc4ad9f05a5bc227 is 50, key is test_row_0/B:col10/1733324170573/Put/seqid=0 2024-12-04T14:56:11,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741880_1056 (size=12629) 2024-12-04T14:56:11,654 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/bb866eff2720498fbc4ad9f05a5bc227 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/bb866eff2720498fbc4ad9f05a5bc227 2024-12-04T14:56:11,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741879_1055 (size=12629) 2024-12-04T14:56:11,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T14:56:11,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:11,658 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-04T14:56:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/d09fd1714a8c4ed995c6f3411663ee58 is 50, key is test_row_0/A:col10/1733324170630/Put/seqid=0 2024-12-04T14:56:11,673 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/B of 7618a4771c68a9e87db105604d8452e1 into bb866eff2720498fbc4ad9f05a5bc227(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:11,673 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:11,673 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/B, priority=12, startTime=1733324171573; duration=0sec 2024-12-04T14:56:11,673 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:11,673 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:B 2024-12-04T14:56:11,673 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:11,675 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:11,675 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/C is initiating minor compaction (all files) 2024-12-04T14:56:11,675 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/C in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:11,676 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2a0e0755bfe44cf7932f35701e457cb1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/29850f880b264ab5b8bdbab6fc5b30aa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a672f41a261f45e9aae22f943e6f0fc1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/65ddc623977947dcbb4ddbd4e0ef79fb] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=47.8 K 2024-12-04T14:56:11,676 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0e0755bfe44cf7932f35701e457cb1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733324167152 2024-12-04T14:56:11,677 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 29850f880b264ab5b8bdbab6fc5b30aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733324169333 2024-12-04T14:56:11,677 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting a672f41a261f45e9aae22f943e6f0fc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733324169405 2024-12-04T14:56:11,678 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ddc623977947dcbb4ddbd4e0ef79fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324170570 2024-12-04T14:56:11,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741881_1057 (size=12151) 2024-12-04T14:56:11,702 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:11,703 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/d09fd1714a8c4ed995c6f3411663ee58 2024-12-04T14:56:11,703 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/84ced081f1fc44d6b56e6ccd19400eea is 50, key is test_row_0/C:col10/1733324170573/Put/seqid=0 2024-12-04T14:56:11,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/c554074b600e414c87ff89cb1a595460 is 50, key is test_row_0/B:col10/1733324170630/Put/seqid=0 2024-12-04T14:56:11,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741882_1058 (size=12629) 2024-12-04T14:56:11,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741883_1059 (size=12151) 2024-12-04T14:56:11,749 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/84ced081f1fc44d6b56e6ccd19400eea as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/84ced081f1fc44d6b56e6ccd19400eea 2024-12-04T14:56:11,749 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/c554074b600e414c87ff89cb1a595460 2024-12-04T14:56:11,760 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/C of 7618a4771c68a9e87db105604d8452e1 into 84ced081f1fc44d6b56e6ccd19400eea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:11,761 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:11,761 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/C, priority=12, startTime=1733324171574; duration=0sec 2024-12-04T14:56:11,761 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:11,761 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:C 2024-12-04T14:56:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/fea73fb269b24fc2a05fa433af389643 is 50, key is test_row_0/C:col10/1733324170630/Put/seqid=0 2024-12-04T14:56:11,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741884_1060 (size=12151) 2024-12-04T14:56:11,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:11,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324231796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324231796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324231797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324231803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:56:11,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324231915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324231915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324231915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,063 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2fd86e2ebdd244d6aada49137fa08274 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2fd86e2ebdd244d6aada49137fa08274 2024-12-04T14:56:12,073 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/A of 7618a4771c68a9e87db105604d8452e1 into 2fd86e2ebdd244d6aada49137fa08274(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:12,073 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:12,073 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/A, priority=12, startTime=1733324171573; duration=0sec 2024-12-04T14:56:12,073 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:12,073 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:A 2024-12-04T14:56:12,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324232126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324232126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324232126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,173 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/fea73fb269b24fc2a05fa433af389643 2024-12-04T14:56:12,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/d09fd1714a8c4ed995c6f3411663ee58 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d09fd1714a8c4ed995c6f3411663ee58 2024-12-04T14:56:12,200 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d09fd1714a8c4ed995c6f3411663ee58, entries=150, sequenceid=236, filesize=11.9 K 2024-12-04T14:56:12,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/c554074b600e414c87ff89cb1a595460 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c554074b600e414c87ff89cb1a595460 2024-12-04T14:56:12,208 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c554074b600e414c87ff89cb1a595460, entries=150, sequenceid=236, filesize=11.9 K 2024-12-04T14:56:12,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/fea73fb269b24fc2a05fa433af389643 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/fea73fb269b24fc2a05fa433af389643 2024-12-04T14:56:12,215 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/fea73fb269b24fc2a05fa433af389643, entries=150, sequenceid=236, filesize=11.9 K 2024-12-04T14:56:12,216 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 7618a4771c68a9e87db105604d8452e1 in 558ms, sequenceid=236, compaction requested=false 2024-12-04T14:56:12,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:12,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:12,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-04T14:56:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-04T14:56:12,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-04T14:56:12,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0230 sec 2024-12-04T14:56:12,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0370 sec 2024-12-04T14:56:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T14:56:12,325 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:12,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-04T14:56:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:56:12,341 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:12,342 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:12,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:12,439 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:12,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-04T14:56:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:56:12,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/0b0b867c63fc4f95b29b44c6f82b3929 is 50, key is test_row_0/A:col10/1733324171801/Put/seqid=0 2024-12-04T14:56:12,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741885_1061 (size=16931) 2024-12-04T14:56:12,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324232486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324232488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324232492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:56:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:12,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324232495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:12,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,499 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_2' in 7618a4771c68a9e87db105604d8452e1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:56:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324232595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324232605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324232605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:56:12,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:12,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:12,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:12,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:56:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324232805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:56:12,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:12,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:12,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:12,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324232815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324232815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324232815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,816 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 7618a4771c68a9e87db105604d8452e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:56:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:12,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/0b0b867c63fc4f95b29b44c6f82b3929 2024-12-04T14:56:12,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/d1ee5cab13c642f3b5fc3b61c8bca6e1 is 50, key is test_row_0/B:col10/1733324171801/Put/seqid=0 2024-12-04T14:56:12,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741886_1062 (size=12151) 2024-12-04T14:56:12,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:56:12,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:12,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:12,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:12,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:56:13,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:56:13,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:13,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:13,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:13,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:13,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:13,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324233115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:13,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:13,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324233125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:13,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:13,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324233125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:13,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:13,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:56:13,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:13,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:13,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/d1ee5cab13c642f3b5fc3b61c8bca6e1 2024-12-04T14:56:13,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/cac51c728bae4ff9ba15060f08572a03 is 50, key is test_row_0/C:col10/1733324171801/Put/seqid=0 2024-12-04T14:56:13,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741887_1063 (size=12151) 2024-12-04T14:56:13,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/cac51c728bae4ff9ba15060f08572a03 2024-12-04T14:56:13,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/0b0b867c63fc4f95b29b44c6f82b3929 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0b0b867c63fc4f95b29b44c6f82b3929 2024-12-04T14:56:13,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0b0b867c63fc4f95b29b44c6f82b3929, entries=250, sequenceid=254, filesize=16.5 K 2024-12-04T14:56:13,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/d1ee5cab13c642f3b5fc3b61c8bca6e1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/d1ee5cab13c642f3b5fc3b61c8bca6e1 2024-12-04T14:56:13,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/d1ee5cab13c642f3b5fc3b61c8bca6e1, entries=150, sequenceid=254, filesize=11.9 K 2024-12-04T14:56:13,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/cac51c728bae4ff9ba15060f08572a03 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/cac51c728bae4ff9ba15060f08572a03 2024-12-04T14:56:13,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/cac51c728bae4ff9ba15060f08572a03, entries=150, sequenceid=254, filesize=11.9 K 2024-12-04T14:56:13,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 7618a4771c68a9e87db105604d8452e1 in 915ms, sequenceid=254, compaction requested=true 2024-12-04T14:56:13,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:13,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:13,354 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:13,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:13,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:13,355 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:13,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:13,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:13,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:13,357 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:13,357 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:13,358 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/B is initiating minor compaction (all files) 2024-12-04T14:56:13,359 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/B in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:13,359 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/A is initiating minor compaction (all files) 2024-12-04T14:56:13,359 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/A in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:13,359 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/bb866eff2720498fbc4ad9f05a5bc227, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c554074b600e414c87ff89cb1a595460, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/d1ee5cab13c642f3b5fc3b61c8bca6e1] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=36.1 K 2024-12-04T14:56:13,359 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2fd86e2ebdd244d6aada49137fa08274, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d09fd1714a8c4ed995c6f3411663ee58, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0b0b867c63fc4f95b29b44c6f82b3929] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=40.7 K 2024-12-04T14:56:13,359 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting bb866eff2720498fbc4ad9f05a5bc227, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324170570 2024-12-04T14:56:13,359 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fd86e2ebdd244d6aada49137fa08274, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324170570 2024-12-04T14:56:13,360 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c554074b600e414c87ff89cb1a595460, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733324170620 2024-12-04T14:56:13,360 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting d09fd1714a8c4ed995c6f3411663ee58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733324170620 2024-12-04T14:56:13,360 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ee5cab13c642f3b5fc3b61c8bca6e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733324171781 2024-12-04T14:56:13,360 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b0b867c63fc4f95b29b44c6f82b3929, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733324171781 2024-12-04T14:56:13,377 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:13,377 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:13,377 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/2ea6de4f27144171ad2d50bba468de47 is 50, key is test_row_0/B:col10/1733324171801/Put/seqid=0 2024-12-04T14:56:13,377 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/733a1407c4074855b3cde6c76eb3e7fa is 50, key is test_row_0/A:col10/1733324171801/Put/seqid=0 2024-12-04T14:56:13,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741888_1064 (size=12731) 2024-12-04T14:56:13,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741889_1065 (size=12731) 2024-12-04T14:56:13,394 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/2ea6de4f27144171ad2d50bba468de47 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2ea6de4f27144171ad2d50bba468de47 2024-12-04T14:56:13,407 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/B of 7618a4771c68a9e87db105604d8452e1 into 2ea6de4f27144171ad2d50bba468de47(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:13,407 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:13,407 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/B, priority=13, startTime=1733324173354; duration=0sec 2024-12-04T14:56:13,407 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:13,407 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:B 2024-12-04T14:56:13,407 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:13,410 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:13,410 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/C is initiating minor compaction (all files) 2024-12-04T14:56:13,410 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/C in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:13,411 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/84ced081f1fc44d6b56e6ccd19400eea, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/fea73fb269b24fc2a05fa433af389643, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/cac51c728bae4ff9ba15060f08572a03] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=36.1 K 2024-12-04T14:56:13,411 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 84ced081f1fc44d6b56e6ccd19400eea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733324170570 2024-12-04T14:56:13,412 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting fea73fb269b24fc2a05fa433af389643, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733324170620 2024-12-04T14:56:13,412 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting cac51c728bae4ff9ba15060f08572a03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733324171781 2024-12-04T14:56:13,420 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#C#compaction#53 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:13,421 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/de9ae6b3bde44e03b9096374477fb9e5 is 50, key is test_row_0/C:col10/1733324171801/Put/seqid=0 2024-12-04T14:56:13,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T14:56:13,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:13,424 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-04T14:56:13,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/d37627d5b93041a78827aeafb7af5731 is 50, key is test_row_0/A:col10/1733324172487/Put/seqid=0 2024-12-04T14:56:13,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741890_1066 (size=12731) 2024-12-04T14:56:13,441 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/de9ae6b3bde44e03b9096374477fb9e5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/de9ae6b3bde44e03b9096374477fb9e5 2024-12-04T14:56:13,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741891_1067 (size=12301) 2024-12-04T14:56:13,449 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/C of 7618a4771c68a9e87db105604d8452e1 into de9ae6b3bde44e03b9096374477fb9e5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:13,449 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:13,449 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/C, priority=13, startTime=1733324173355; duration=0sec 2024-12-04T14:56:13,450 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:13,450 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:C 2024-12-04T14:56:13,451 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/d37627d5b93041a78827aeafb7af5731 2024-12-04T14:56:13,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/c1ef008bc45d4e50b093ce3bf7ace4a5 is 50, key is test_row_0/B:col10/1733324172487/Put/seqid=0 2024-12-04T14:56:13,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741892_1068 (size=12301) 2024-12-04T14:56:13,467 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/c1ef008bc45d4e50b093ce3bf7ace4a5 2024-12-04T14:56:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:56:13,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/52bb6e9bc5114fd4aed5d472a2e2b82a is 50, key is test_row_0/C:col10/1733324172487/Put/seqid=0 2024-12-04T14:56:13,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741893_1069 (size=12301) 2024-12-04T14:56:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:13,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:13,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324233653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:13,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324233654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324233657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:13,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:13,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324233765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324233766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:13,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324233766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:13,801 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/733a1407c4074855b3cde6c76eb3e7fa as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/733a1407c4074855b3cde6c76eb3e7fa 2024-12-04T14:56:13,809 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/A of 7618a4771c68a9e87db105604d8452e1 into 733a1407c4074855b3cde6c76eb3e7fa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:13,809 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:13,809 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/A, priority=13, startTime=1733324173354; duration=0sec 2024-12-04T14:56:13,809 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:13,809 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:A 2024-12-04T14:56:13,889 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/52bb6e9bc5114fd4aed5d472a2e2b82a 2024-12-04T14:56:13,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/d37627d5b93041a78827aeafb7af5731 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d37627d5b93041a78827aeafb7af5731 2024-12-04T14:56:13,902 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d37627d5b93041a78827aeafb7af5731, entries=150, sequenceid=276, filesize=12.0 K 2024-12-04T14:56:13,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/c1ef008bc45d4e50b093ce3bf7ace4a5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c1ef008bc45d4e50b093ce3bf7ace4a5 2024-12-04T14:56:13,910 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c1ef008bc45d4e50b093ce3bf7ace4a5, entries=150, sequenceid=276, filesize=12.0 K 2024-12-04T14:56:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/52bb6e9bc5114fd4aed5d472a2e2b82a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/52bb6e9bc5114fd4aed5d472a2e2b82a 2024-12-04T14:56:13,918 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/52bb6e9bc5114fd4aed5d472a2e2b82a, entries=150, sequenceid=276, filesize=12.0 K 2024-12-04T14:56:13,919 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 7618a4771c68a9e87db105604d8452e1 in 495ms, sequenceid=276, compaction requested=false 2024-12-04T14:56:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-04T14:56:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-04T14:56:13,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-04T14:56:13,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5800 sec 2024-12-04T14:56:13,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6000 sec 2024-12-04T14:56:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:13,978 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:13,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-04T14:56:13,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/0d77f2e622bb47e2b33aca616539183f is 50, key is test_row_0/A:col10/1733324173976/Put/seqid=0 2024-12-04T14:56:14,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324234025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324234025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324234028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741894_1070 (size=12301) 2024-12-04T14:56:14,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/0d77f2e622bb47e2b33aca616539183f 2024-12-04T14:56:14,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/2e38a60d876d4430936e16f5bd112f8a is 50, key is test_row_0/B:col10/1733324173976/Put/seqid=0 2024-12-04T14:56:14,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741895_1071 (size=12301) 2024-12-04T14:56:14,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/2e38a60d876d4430936e16f5bd112f8a 2024-12-04T14:56:14,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/e145b3b41d6b422689245760f4a382de is 50, key is test_row_0/C:col10/1733324173976/Put/seqid=0 2024-12-04T14:56:14,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741896_1072 (size=12301) 2024-12-04T14:56:14,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324234135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324234135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324234135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324234345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324234345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324234346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T14:56:14,485 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:14,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-04T14:56:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:56:14,488 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:14,489 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:14,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:14,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/e145b3b41d6b422689245760f4a382de 2024-12-04T14:56:14,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/0d77f2e622bb47e2b33aca616539183f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0d77f2e622bb47e2b33aca616539183f 2024-12-04T14:56:14,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0d77f2e622bb47e2b33aca616539183f, entries=150, sequenceid=294, filesize=12.0 K 2024-12-04T14:56:14,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/2e38a60d876d4430936e16f5bd112f8a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2e38a60d876d4430936e16f5bd112f8a 2024-12-04T14:56:14,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2e38a60d876d4430936e16f5bd112f8a, entries=150, sequenceid=294, filesize=12.0 K 2024-12-04T14:56:14,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/e145b3b41d6b422689245760f4a382de as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/e145b3b41d6b422689245760f4a382de 2024-12-04T14:56:14,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/e145b3b41d6b422689245760f4a382de, entries=150, sequenceid=294, filesize=12.0 K 2024-12-04T14:56:14,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 7618a4771c68a9e87db105604d8452e1 in 570ms, sequenceid=294, compaction requested=true 2024-12-04T14:56:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T14:56:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-04T14:56:14,550 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:14,550 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:14,552 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:14,553 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/C is initiating minor compaction (all files) 2024-12-04T14:56:14,553 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/C in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:14,553 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/de9ae6b3bde44e03b9096374477fb9e5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/52bb6e9bc5114fd4aed5d472a2e2b82a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/e145b3b41d6b422689245760f4a382de] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=36.5 K 2024-12-04T14:56:14,553 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:14,553 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/A is initiating minor compaction (all files) 2024-12-04T14:56:14,553 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/A in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:14,554 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/733a1407c4074855b3cde6c76eb3e7fa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d37627d5b93041a78827aeafb7af5731, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0d77f2e622bb47e2b33aca616539183f] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=36.5 K 2024-12-04T14:56:14,554 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 733a1407c4074855b3cde6c76eb3e7fa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733324171781 2024-12-04T14:56:14,554 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting de9ae6b3bde44e03b9096374477fb9e5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733324171781 2024-12-04T14:56:14,555 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting d37627d5b93041a78827aeafb7af5731, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733324172462 2024-12-04T14:56:14,555 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 52bb6e9bc5114fd4aed5d472a2e2b82a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733324172462 2024-12-04T14:56:14,557 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d77f2e622bb47e2b33aca616539183f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733324173648 2024-12-04T14:56:14,558 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e145b3b41d6b422689245760f4a382de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733324173648 2024-12-04T14:56:14,577 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:14,577 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/529ae962b6404e4e868e08f9a10b1d41 is 50, key is test_row_0/A:col10/1733324173976/Put/seqid=0 2024-12-04T14:56:14,582 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#C#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:14,583 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/d80c4c7319b84098b4544532a8f9443c is 50, key is test_row_0/C:col10/1733324173976/Put/seqid=0 2024-12-04T14:56:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:56:14,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741897_1073 (size=12983) 2024-12-04T14:56:14,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741898_1074 (size=12983) 2024-12-04T14:56:14,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T14:56:14,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:14,642 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-04T14:56:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/29b78613923245f79db8844a3b079f6f is 50, key is test_row_0/A:col10/1733324174010/Put/seqid=0 2024-12-04T14:56:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:14,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:14,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741899_1075 (size=12301) 2024-12-04T14:56:14,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324234670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324234672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324234676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324234785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324234785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324234785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:56:14,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324234825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,831 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 7618a4771c68a9e87db105604d8452e1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:56:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324234995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324234995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324234996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:15,008 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/529ae962b6404e4e868e08f9a10b1d41 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/529ae962b6404e4e868e08f9a10b1d41 2024-12-04T14:56:15,015 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/d80c4c7319b84098b4544532a8f9443c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/d80c4c7319b84098b4544532a8f9443c 2024-12-04T14:56:15,015 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/A of 7618a4771c68a9e87db105604d8452e1 into 529ae962b6404e4e868e08f9a10b1d41(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:15,015 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:15,015 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/A, priority=13, startTime=1733324174549; duration=0sec 2024-12-04T14:56:15,018 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:15,019 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:A 2024-12-04T14:56:15,019 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:15,020 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:15,020 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/B is initiating minor compaction (all files) 2024-12-04T14:56:15,020 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/B in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:15,020 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2ea6de4f27144171ad2d50bba468de47, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c1ef008bc45d4e50b093ce3bf7ace4a5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2e38a60d876d4430936e16f5bd112f8a] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=36.5 K 2024-12-04T14:56:15,021 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ea6de4f27144171ad2d50bba468de47, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733324171781 2024-12-04T14:56:15,021 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1ef008bc45d4e50b093ce3bf7ace4a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733324172462 2024-12-04T14:56:15,022 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e38a60d876d4430936e16f5bd112f8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733324173648 2024-12-04T14:56:15,026 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/C of 7618a4771c68a9e87db105604d8452e1 into d80c4c7319b84098b4544532a8f9443c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:15,026 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:15,026 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/C, priority=13, startTime=1733324174549; duration=0sec 2024-12-04T14:56:15,027 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:15,027 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:C 2024-12-04T14:56:15,029 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#B#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:15,029 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/b137e07adb0d44afa9b7f05bc97d191b is 50, key is test_row_0/B:col10/1733324173976/Put/seqid=0 2024-12-04T14:56:15,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741900_1076 (size=12983) 2024-12-04T14:56:15,059 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/29b78613923245f79db8844a3b079f6f 2024-12-04T14:56:15,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/25d9f3f6f2d743d79f2c9a1c1ce4c2ac is 50, key is test_row_0/B:col10/1733324174010/Put/seqid=0 2024-12-04T14:56:15,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741901_1077 (size=12301) 2024-12-04T14:56:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:56:15,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324235305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:15,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324235305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:15,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324235306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:15,448 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/b137e07adb0d44afa9b7f05bc97d191b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b137e07adb0d44afa9b7f05bc97d191b 2024-12-04T14:56:15,454 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/B of 7618a4771c68a9e87db105604d8452e1 into b137e07adb0d44afa9b7f05bc97d191b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:15,454 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:15,454 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/B, priority=13, startTime=1733324174549; duration=0sec 2024-12-04T14:56:15,454 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:15,454 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:B 2024-12-04T14:56:15,477 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/25d9f3f6f2d743d79f2c9a1c1ce4c2ac 2024-12-04T14:56:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/9a15bb696487401caac65f0d85476f8c is 50, key is test_row_0/C:col10/1733324174010/Put/seqid=0 2024-12-04T14:56:15,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741902_1078 (size=12301) 2024-12-04T14:56:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:56:15,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:15,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324235815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324235815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:15,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324235816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:15,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:15,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:15,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:15,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:15,900 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/9a15bb696487401caac65f0d85476f8c 2024-12-04T14:56:15,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/29b78613923245f79db8844a3b079f6f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/29b78613923245f79db8844a3b079f6f 2024-12-04T14:56:15,932 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/29b78613923245f79db8844a3b079f6f, entries=150, sequenceid=315, filesize=12.0 K 2024-12-04T14:56:15,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/25d9f3f6f2d743d79f2c9a1c1ce4c2ac as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/25d9f3f6f2d743d79f2c9a1c1ce4c2ac 2024-12-04T14:56:15,943 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/25d9f3f6f2d743d79f2c9a1c1ce4c2ac, entries=150, sequenceid=315, filesize=12.0 K 2024-12-04T14:56:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/9a15bb696487401caac65f0d85476f8c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/9a15bb696487401caac65f0d85476f8c 2024-12-04T14:56:15,952 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/9a15bb696487401caac65f0d85476f8c, entries=150, sequenceid=315, filesize=12.0 K 2024-12-04T14:56:15,953 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 7618a4771c68a9e87db105604d8452e1 in 1311ms, sequenceid=315, compaction requested=false 2024-12-04T14:56:15,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:15,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:15,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-04T14:56:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-04T14:56:15,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-04T14:56:15,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-12-04T14:56:15,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4730 sec 2024-12-04T14:56:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T14:56:16,636 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:16,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-04T14:56:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:56:16,640 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:16,641 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:16,641 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:56:16,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T14:56:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:16,794 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-04T14:56:16,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/1b278d8f4b684d12bca05076f3f51398 is 50, key is test_row_0/A:col10/1733324174674/Put/seqid=0 2024-12-04T14:56:16,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741903_1079 (size=12301) 2024-12-04T14:56:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:16,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:16,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324236855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:16,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324236874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324236874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:56:16,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324236985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324236985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324236985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:17,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324237195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:17,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324237195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324237195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:17,206 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/1b278d8f4b684d12bca05076f3f51398 2024-12-04T14:56:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/eef9c001ed894b3b8cdd00cb777c534c is 50, key is test_row_0/B:col10/1733324174674/Put/seqid=0 2024-12-04T14:56:17,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741904_1080 (size=12301) 2024-12-04T14:56:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:56:17,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:17,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324237505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:17,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324237505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:17,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324237505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:17,622 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/eef9c001ed894b3b8cdd00cb777c534c 2024-12-04T14:56:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/737f54b530764518b1a77f1e1c0bd65c is 50, key is test_row_0/C:col10/1733324174674/Put/seqid=0 2024-12-04T14:56:17,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741905_1081 (size=12301) 2024-12-04T14:56:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:56:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324238015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324238015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324238015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:18,041 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/737f54b530764518b1a77f1e1c0bd65c 2024-12-04T14:56:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/1b278d8f4b684d12bca05076f3f51398 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1b278d8f4b684d12bca05076f3f51398 2024-12-04T14:56:18,054 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1b278d8f4b684d12bca05076f3f51398, entries=150, sequenceid=333, filesize=12.0 K 2024-12-04T14:56:18,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/eef9c001ed894b3b8cdd00cb777c534c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/eef9c001ed894b3b8cdd00cb777c534c 2024-12-04T14:56:18,063 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/eef9c001ed894b3b8cdd00cb777c534c, entries=150, sequenceid=333, filesize=12.0 K 2024-12-04T14:56:18,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/737f54b530764518b1a77f1e1c0bd65c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/737f54b530764518b1a77f1e1c0bd65c 2024-12-04T14:56:18,070 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/737f54b530764518b1a77f1e1c0bd65c, entries=150, sequenceid=333, filesize=12.0 K 2024-12-04T14:56:18,071 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 7618a4771c68a9e87db105604d8452e1 in 1277ms, sequenceid=333, compaction requested=true 2024-12-04T14:56:18,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:18,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:18,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-04T14:56:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-04T14:56:18,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-04T14:56:18,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4320 sec 2024-12-04T14:56:18,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4400 sec 2024-12-04T14:56:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T14:56:18,785 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:18,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-04T14:56:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T14:56:18,790 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:18,791 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:18,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:18,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-04T14:56:18,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/1eae467861c84f3894702d04fbfb75fd is 50, key is test_row_0/A:col10/1733324176858/Put/seqid=0 2024-12-04T14:56:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T14:56:18,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741906_1082 (size=14741) 2024-12-04T14:56:18,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/1eae467861c84f3894702d04fbfb75fd 2024-12-04T14:56:18,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/32a3a4e2045440ae8997bffcfb01cf7c is 50, key is test_row_0/B:col10/1733324176858/Put/seqid=0 2024-12-04T14:56:18,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T14:56:18,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:18,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:18,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:18,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:18,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:18,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741907_1083 (size=12301) 2024-12-04T14:56:18,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324238941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:18,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/32a3a4e2045440ae8997bffcfb01cf7c 2024-12-04T14:56:18,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/22f7cb8041ef4adb989b2d6f02ba291c is 50, key is test_row_0/C:col10/1733324176858/Put/seqid=0 2024-12-04T14:56:18,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741908_1084 (size=12301) 2024-12-04T14:56:18,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/22f7cb8041ef4adb989b2d6f02ba291c 2024-12-04T14:56:18,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/1eae467861c84f3894702d04fbfb75fd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1eae467861c84f3894702d04fbfb75fd 2024-12-04T14:56:18,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1eae467861c84f3894702d04fbfb75fd, entries=200, sequenceid=355, filesize=14.4 K 2024-12-04T14:56:18,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/32a3a4e2045440ae8997bffcfb01cf7c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/32a3a4e2045440ae8997bffcfb01cf7c 2024-12-04T14:56:19,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/32a3a4e2045440ae8997bffcfb01cf7c, entries=150, sequenceid=355, filesize=12.0 K 2024-12-04T14:56:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/22f7cb8041ef4adb989b2d6f02ba291c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/22f7cb8041ef4adb989b2d6f02ba291c 2024-12-04T14:56:19,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/22f7cb8041ef4adb989b2d6f02ba291c, entries=150, sequenceid=355, filesize=12.0 K 2024-12-04T14:56:19,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 7618a4771c68a9e87db105604d8452e1 in 138ms, sequenceid=355, compaction requested=true 2024-12-04T14:56:19,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:19,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:19,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:19,017 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:19,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:19,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:19,017 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:19,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:19,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:19,020 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:19,020 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/B is initiating minor compaction (all files) 2024-12-04T14:56:19,020 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/B in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,020 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b137e07adb0d44afa9b7f05bc97d191b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/25d9f3f6f2d743d79f2c9a1c1ce4c2ac, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/eef9c001ed894b3b8cdd00cb777c534c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/32a3a4e2045440ae8997bffcfb01cf7c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=48.7 K 2024-12-04T14:56:19,021 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b137e07adb0d44afa9b7f05bc97d191b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733324173648 2024-12-04T14:56:19,021 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:19,021 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/A is initiating minor compaction (all files) 2024-12-04T14:56:19,021 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/A in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,021 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/529ae962b6404e4e868e08f9a10b1d41, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/29b78613923245f79db8844a3b079f6f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1b278d8f4b684d12bca05076f3f51398, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1eae467861c84f3894702d04fbfb75fd] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=51.1 K 2024-12-04T14:56:19,022 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d9f3f6f2d743d79f2c9a1c1ce4c2ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733324174010 2024-12-04T14:56:19,022 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 529ae962b6404e4e868e08f9a10b1d41, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733324173648 2024-12-04T14:56:19,022 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting eef9c001ed894b3b8cdd00cb777c534c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733324174661 2024-12-04T14:56:19,023 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 32a3a4e2045440ae8997bffcfb01cf7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733324176858 2024-12-04T14:56:19,023 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b78613923245f79db8844a3b079f6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733324174010 2024-12-04T14:56:19,024 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b278d8f4b684d12bca05076f3f51398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733324174661 2024-12-04T14:56:19,025 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eae467861c84f3894702d04fbfb75fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733324176854 2024-12-04T14:56:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:19,027 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:19,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-04T14:56:19,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2ed00bd2d1fb4593b25cb1151fb00e53 is 50, key is test_row_0/A:col10/1733324179026/Put/seqid=0 2024-12-04T14:56:19,043 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:19,044 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/ee8b806511564e7b843baf9441816617 is 50, key is test_row_0/B:col10/1733324176858/Put/seqid=0 2024-12-04T14:56:19,045 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#A#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:19,046 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/dca78ba8ca8c44948e1390591d4e728a is 50, key is test_row_0/A:col10/1733324176858/Put/seqid=0 2024-12-04T14:56:19,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741910_1086 (size=13119) 2024-12-04T14:56:19,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741909_1085 (size=14741) 2024-12-04T14:56:19,068 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/ee8b806511564e7b843baf9441816617 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/ee8b806511564e7b843baf9441816617 2024-12-04T14:56:19,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324239065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324239070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,077 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/B of 7618a4771c68a9e87db105604d8452e1 into ee8b806511564e7b843baf9441816617(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:19,078 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324239071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,078 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/B, priority=12, startTime=1733324179017; duration=0sec 2024-12-04T14:56:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,078 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,078 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:B 2024-12-04T14:56:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,078 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:19,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324239073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,081 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:19,082 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/C is initiating minor compaction (all files) 2024-12-04T14:56:19,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741911_1087 (size=13119) 2024-12-04T14:56:19,082 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/C in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,082 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/d80c4c7319b84098b4544532a8f9443c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/9a15bb696487401caac65f0d85476f8c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/737f54b530764518b1a77f1e1c0bd65c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/22f7cb8041ef4adb989b2d6f02ba291c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=48.7 K 2024-12-04T14:56:19,082 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting d80c4c7319b84098b4544532a8f9443c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733324173648 2024-12-04T14:56:19,083 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a15bb696487401caac65f0d85476f8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733324174010 2024-12-04T14:56:19,083 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 737f54b530764518b1a77f1e1c0bd65c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733324174661 2024-12-04T14:56:19,083 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 22f7cb8041ef4adb989b2d6f02ba291c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733324176858 2024-12-04T14:56:19,096 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#C#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:19,097 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/00486d85780342d395b8ed54e1dd7e6f is 50, key is test_row_0/C:col10/1733324176858/Put/seqid=0 2024-12-04T14:56:19,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T14:56:19,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:19,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T14:56:19,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741912_1088 (size=13119) 2024-12-04T14:56:19,146 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/00486d85780342d395b8ed54e1dd7e6f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/00486d85780342d395b8ed54e1dd7e6f 2024-12-04T14:56:19,155 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/C of 7618a4771c68a9e87db105604d8452e1 into 00486d85780342d395b8ed54e1dd7e6f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:19,155 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:19,155 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/C, priority=12, startTime=1733324179017; duration=0sec 2024-12-04T14:56:19,155 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:19,155 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:C 2024-12-04T14:56:19,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324239184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324239185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324239185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324239185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T14:56:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:19,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324239395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324239395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324239395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324239395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T14:56:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T14:56:19,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2ed00bd2d1fb4593b25cb1151fb00e53 2024-12-04T14:56:19,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/be7c03813aa04252bdd414efb487c7c0 is 50, key is test_row_0/B:col10/1733324179026/Put/seqid=0 2024-12-04T14:56:19,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741913_1089 (size=12301) 2024-12-04T14:56:19,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/be7c03813aa04252bdd414efb487c7c0 2024-12-04T14:56:19,491 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/dca78ba8ca8c44948e1390591d4e728a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/dca78ba8ca8c44948e1390591d4e728a 2024-12-04T14:56:19,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/a195ee52fa014f8d8de99c308e0ef719 is 50, key is test_row_0/C:col10/1733324179026/Put/seqid=0 2024-12-04T14:56:19,498 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/A of 7618a4771c68a9e87db105604d8452e1 into dca78ba8ca8c44948e1390591d4e728a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:19,498 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:19,498 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/A, priority=12, startTime=1733324179017; duration=0sec 2024-12-04T14:56:19,498 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:19,498 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:A 2024-12-04T14:56:19,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741914_1090 (size=12301) 2024-12-04T14:56:19,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T14:56:19,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:19,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324239705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324239705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324239705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324239706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:19,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T14:56:19,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:19,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T14:56:19,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:19,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:19,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:19,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/a195ee52fa014f8d8de99c308e0ef719 2024-12-04T14:56:19,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2ed00bd2d1fb4593b25cb1151fb00e53 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ed00bd2d1fb4593b25cb1151fb00e53 2024-12-04T14:56:19,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ed00bd2d1fb4593b25cb1151fb00e53, entries=200, sequenceid=370, filesize=14.4 K 2024-12-04T14:56:19,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/be7c03813aa04252bdd414efb487c7c0 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/be7c03813aa04252bdd414efb487c7c0 2024-12-04T14:56:19,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/be7c03813aa04252bdd414efb487c7c0, entries=150, sequenceid=370, filesize=12.0 K 2024-12-04T14:56:19,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/a195ee52fa014f8d8de99c308e0ef719 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a195ee52fa014f8d8de99c308e0ef719 2024-12-04T14:56:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T14:56:19,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a195ee52fa014f8d8de99c308e0ef719, entries=150, sequenceid=370, filesize=12.0 K 2024-12-04T14:56:19,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 7618a4771c68a9e87db105604d8452e1 in 901ms, sequenceid=370, compaction requested=false 2024-12-04T14:56:19,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:20,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T14:56:20,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:20,019 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-04T14:56:20,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/25bac21c32a544969bc7f1087f0dadc1 is 50, key is test_row_0/A:col10/1733324179070/Put/seqid=0 2024-12-04T14:56:20,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741915_1091 (size=12301) 2024-12-04T14:56:20,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324240222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324240224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324240225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324240226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324240335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324240335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324240335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324240335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,429 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/25bac21c32a544969bc7f1087f0dadc1 2024-12-04T14:56:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/1afe723267994647989ea01d0ab18fdf is 50, key is test_row_0/B:col10/1733324179070/Put/seqid=0 2024-12-04T14:56:20,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741916_1092 (size=12301) 2024-12-04T14:56:20,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324240545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324240545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324240545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324240545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,841 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/1afe723267994647989ea01d0ab18fdf 2024-12-04T14:56:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/2909e479e47148d9ac0157e59f596430 is 50, key is test_row_0/C:col10/1733324179070/Put/seqid=0 2024-12-04T14:56:20,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741917_1093 (size=12301) 2024-12-04T14:56:20,854 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/2909e479e47148d9ac0157e59f596430 2024-12-04T14:56:20,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324240855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324240855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324240855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324240855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:20,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/25bac21c32a544969bc7f1087f0dadc1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/25bac21c32a544969bc7f1087f0dadc1 2024-12-04T14:56:20,865 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/25bac21c32a544969bc7f1087f0dadc1, entries=150, sequenceid=395, filesize=12.0 K 2024-12-04T14:56:20,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/1afe723267994647989ea01d0ab18fdf as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/1afe723267994647989ea01d0ab18fdf 2024-12-04T14:56:20,873 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/1afe723267994647989ea01d0ab18fdf, entries=150, sequenceid=395, filesize=12.0 K 2024-12-04T14:56:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/2909e479e47148d9ac0157e59f596430 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2909e479e47148d9ac0157e59f596430 2024-12-04T14:56:20,881 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2909e479e47148d9ac0157e59f596430, entries=150, sequenceid=395, filesize=12.0 K 2024-12-04T14:56:20,882 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 7618a4771c68a9e87db105604d8452e1 in 864ms, sequenceid=395, compaction requested=true 2024-12-04T14:56:20,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:20,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:20,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-04T14:56:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-04T14:56:20,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-04T14:56:20,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0930 sec 2024-12-04T14:56:20,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1010 sec 2024-12-04T14:56:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T14:56:20,935 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:20,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-04T14:56:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T14:56:20,938 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:20,939 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:20,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T14:56:21,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-04T14:56:21,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:21,092 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-04T14:56:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2df62f4885e34f2c9786a7a3d159d99f is 50, key is test_row_0/A:col10/1733324180224/Put/seqid=0 2024-12-04T14:56:21,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741918_1094 (size=12301) 2024-12-04T14:56:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T14:56:21,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:21,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:21,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324241410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324241411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324241414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324241415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,502 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2df62f4885e34f2c9786a7a3d159d99f 2024-12-04T14:56:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/90853c57068f401286c4aac4b66db553 is 50, key is test_row_0/B:col10/1733324180224/Put/seqid=0 2024-12-04T14:56:21,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741919_1095 (size=12301) 2024-12-04T14:56:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324241525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324241525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324241525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324241525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T14:56:21,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324241735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324241735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324241735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324241735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:21,915 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/90853c57068f401286c4aac4b66db553 2024-12-04T14:56:21,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/662746d035c14e9fbb024e5eb55a537b is 50, key is test_row_0/C:col10/1733324180224/Put/seqid=0 2024-12-04T14:56:21,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741920_1096 (size=12301) 2024-12-04T14:56:22,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324242045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324242045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324242045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324242045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T14:56:22,334 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/662746d035c14e9fbb024e5eb55a537b 2024-12-04T14:56:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/2df62f4885e34f2c9786a7a3d159d99f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2df62f4885e34f2c9786a7a3d159d99f 2024-12-04T14:56:22,345 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2df62f4885e34f2c9786a7a3d159d99f, entries=150, sequenceid=409, filesize=12.0 K 2024-12-04T14:56:22,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/90853c57068f401286c4aac4b66db553 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/90853c57068f401286c4aac4b66db553 2024-12-04T14:56:22,351 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/90853c57068f401286c4aac4b66db553, entries=150, sequenceid=409, filesize=12.0 K 2024-12-04T14:56:22,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/662746d035c14e9fbb024e5eb55a537b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/662746d035c14e9fbb024e5eb55a537b 2024-12-04T14:56:22,358 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/662746d035c14e9fbb024e5eb55a537b, entries=150, sequenceid=409, filesize=12.0 K 2024-12-04T14:56:22,359 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 7618a4771c68a9e87db105604d8452e1 in 1267ms, sequenceid=409, compaction requested=true 2024-12-04T14:56:22,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:22,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:22,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-04T14:56:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-04T14:56:22,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-04T14:56:22,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2024-12-04T14:56:22,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4280 sec 2024-12-04T14:56:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:22,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-04T14:56:22,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/e0db9f6e0d7b451ab4d7ab4570308f1c is 50, key is test_row_0/A:col10/1733324181409/Put/seqid=0 2024-12-04T14:56:22,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324242564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741921_1097 (size=14741) 2024-12-04T14:56:22,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324242567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324242567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324242568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324242571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324242675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324242684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324242684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324242685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324242685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324242884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324242894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324242895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324242895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324242895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:22,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/e0db9f6e0d7b451ab4d7ab4570308f1c 2024-12-04T14:56:22,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/6a94600648754014a1be0c382b0d8e7f is 50, key is test_row_0/B:col10/1733324181409/Put/seqid=0 2024-12-04T14:56:22,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741922_1098 (size=12301) 2024-12-04T14:56:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T14:56:23,086 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:23,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-04T14:56:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T14:56:23,088 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:23,090 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:23,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T14:56:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324243195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:23,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324243204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:23,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324243205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:23,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324243205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324243205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:23,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-04T14:56:23,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:23,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/6a94600648754014a1be0c382b0d8e7f 2024-12-04T14:56:23,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/485eae8f0ad842209281e60b86b1fa04 is 50, key is test_row_0/C:col10/1733324181409/Put/seqid=0 2024-12-04T14:56:23,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-04T14:56:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741923_1099 (size=12301) 2024-12-04T14:56:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T14:56:23,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-04T14:56:23,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:23,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,587 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-04T14:56:23,587 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-04T14:56:23,587 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:23,587 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:23,589 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-04T14:56:23,589 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:23,592 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:23,592 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:23,592 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:23,593 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:23,593 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:23,593 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:23,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:23,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:23,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:23,596 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-04T14:56:23,596 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:23,596 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:23,596 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:23,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:23,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-04T14:56:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35214 deadline: 1733324243705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:23,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:23,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:23,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T14:56:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1733324243716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35226 deadline: 1733324243716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35222 deadline: 1733324243716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35212 deadline: 1733324243716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:23,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/485eae8f0ad842209281e60b86b1fa04 2024-12-04T14:56:23,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/e0db9f6e0d7b451ab4d7ab4570308f1c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e0db9f6e0d7b451ab4d7ab4570308f1c 2024-12-04T14:56:23,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e0db9f6e0d7b451ab4d7ab4570308f1c, entries=200, sequenceid=432, filesize=14.4 K 2024-12-04T14:56:23,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/6a94600648754014a1be0c382b0d8e7f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/6a94600648754014a1be0c382b0d8e7f 2024-12-04T14:56:23,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/6a94600648754014a1be0c382b0d8e7f, entries=150, sequenceid=432, filesize=12.0 K 2024-12-04T14:56:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/485eae8f0ad842209281e60b86b1fa04 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/485eae8f0ad842209281e60b86b1fa04 2024-12-04T14:56:23,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/485eae8f0ad842209281e60b86b1fa04, entries=150, sequenceid=432, filesize=12.0 K 2024-12-04T14:56:23,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7618a4771c68a9e87db105604d8452e1 in 1270ms, sequenceid=432, compaction requested=true 2024-12-04T14:56:23,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:23,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:23,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:23,827 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:56:23,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:23,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:23,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7618a4771c68a9e87db105604d8452e1:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:23,827 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:56:23,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:23,829 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62323 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:56:23,829 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67203 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:56:23,829 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/B is initiating minor compaction (all files) 2024-12-04T14:56:23,829 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/A is initiating minor compaction (all files) 2024-12-04T14:56:23,829 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/B in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,829 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/A in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,829 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/dca78ba8ca8c44948e1390591d4e728a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ed00bd2d1fb4593b25cb1151fb00e53, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/25bac21c32a544969bc7f1087f0dadc1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2df62f4885e34f2c9786a7a3d159d99f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e0db9f6e0d7b451ab4d7ab4570308f1c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=65.6 K 2024-12-04T14:56:23,829 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/ee8b806511564e7b843baf9441816617, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/be7c03813aa04252bdd414efb487c7c0, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/1afe723267994647989ea01d0ab18fdf, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/90853c57068f401286c4aac4b66db553, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/6a94600648754014a1be0c382b0d8e7f] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=60.9 K 2024-12-04T14:56:23,829 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca78ba8ca8c44948e1390591d4e728a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733324176858 2024-12-04T14:56:23,829 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting ee8b806511564e7b843baf9441816617, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733324176858 2024-12-04T14:56:23,830 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ed00bd2d1fb4593b25cb1151fb00e53, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733324178923 2024-12-04T14:56:23,830 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting be7c03813aa04252bdd414efb487c7c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733324178923 2024-12-04T14:56:23,830 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25bac21c32a544969bc7f1087f0dadc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733324179060 2024-12-04T14:56:23,830 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 1afe723267994647989ea01d0ab18fdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733324179060 2024-12-04T14:56:23,830 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2df62f4885e34f2c9786a7a3d159d99f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733324180223 2024-12-04T14:56:23,830 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 90853c57068f401286c4aac4b66db553, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733324180223 2024-12-04T14:56:23,830 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0db9f6e0d7b451ab4d7ab4570308f1c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733324181409 2024-12-04T14:56:23,830 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a94600648754014a1be0c382b0d8e7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733324181409 2024-12-04T14:56:23,841 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:23,841 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/e6ea2b2954cc429ca8584a46f0542c72 is 50, key is test_row_0/A:col10/1733324181409/Put/seqid=0 2024-12-04T14:56:23,842 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:23,842 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/e344a551a4424b279cf0e3d5a247446a is 50, key is test_row_0/B:col10/1733324181409/Put/seqid=0 2024-12-04T14:56:23,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741924_1100 (size=13289) 2024-12-04T14:56:23,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741925_1101 (size=13289) 2024-12-04T14:56:23,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-04T14:56:23,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:23,857 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-04T14:56:23,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/114cf1c83d764784bbeff74cb7663def is 50, key is test_row_0/A:col10/1733324182559/Put/seqid=0 2024-12-04T14:56:23,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741926_1102 (size=12301) 2024-12-04T14:56:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T14:56:24,257 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/e6ea2b2954cc429ca8584a46f0542c72 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e6ea2b2954cc429ca8584a46f0542c72 2024-12-04T14:56:24,257 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/e344a551a4424b279cf0e3d5a247446a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/e344a551a4424b279cf0e3d5a247446a 2024-12-04T14:56:24,264 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/B of 7618a4771c68a9e87db105604d8452e1 into e344a551a4424b279cf0e3d5a247446a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:24,264 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/A of 7618a4771c68a9e87db105604d8452e1 into e6ea2b2954cc429ca8584a46f0542c72(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:24,264 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:24,264 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:24,264 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/A, priority=11, startTime=1733324183827; duration=0sec 2024-12-04T14:56:24,264 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/B, priority=11, startTime=1733324183827; duration=0sec 2024-12-04T14:56:24,264 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:24,264 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:A 2024-12-04T14:56:24,264 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:24,264 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:B 2024-12-04T14:56:24,264 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T14:56:24,266 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62323 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T14:56:24,266 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 7618a4771c68a9e87db105604d8452e1/C is initiating minor compaction (all files) 2024-12-04T14:56:24,266 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7618a4771c68a9e87db105604d8452e1/C in TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:24,266 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/114cf1c83d764784bbeff74cb7663def 2024-12-04T14:56:24,266 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/00486d85780342d395b8ed54e1dd7e6f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a195ee52fa014f8d8de99c308e0ef719, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2909e479e47148d9ac0157e59f596430, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/662746d035c14e9fbb024e5eb55a537b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/485eae8f0ad842209281e60b86b1fa04] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp, totalSize=60.9 K 2024-12-04T14:56:24,267 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00486d85780342d395b8ed54e1dd7e6f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733324176858 2024-12-04T14:56:24,267 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting a195ee52fa014f8d8de99c308e0ef719, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733324178923 2024-12-04T14:56:24,268 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2909e479e47148d9ac0157e59f596430, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733324179060 2024-12-04T14:56:24,268 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 662746d035c14e9fbb024e5eb55a537b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733324180223 2024-12-04T14:56:24,268 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 485eae8f0ad842209281e60b86b1fa04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733324181409 2024-12-04T14:56:24,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/80397ad544b54833bf330abed5536eb3 is 50, key is test_row_0/B:col10/1733324182559/Put/seqid=0 2024-12-04T14:56:24,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741927_1103 (size=12301) 2024-12-04T14:56:24,295 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7618a4771c68a9e87db105604d8452e1#C#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:24,296 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/9daf2d12f8f04a5e91a8d93df4218577 is 50, key is test_row_0/C:col10/1733324181409/Put/seqid=0 2024-12-04T14:56:24,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741928_1104 (size=13289) 2024-12-04T14:56:24,696 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/80397ad544b54833bf330abed5536eb3 2024-12-04T14:56:24,709 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/9daf2d12f8f04a5e91a8d93df4218577 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/9daf2d12f8f04a5e91a8d93df4218577 2024-12-04T14:56:24,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/eea75abc7d7d4f54a052e53c6c5503a1 is 50, key is test_row_0/C:col10/1733324182559/Put/seqid=0 2024-12-04T14:56:24,715 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7618a4771c68a9e87db105604d8452e1/C of 7618a4771c68a9e87db105604d8452e1 into 9daf2d12f8f04a5e91a8d93df4218577(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:24,716 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:24,716 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., storeName=7618a4771c68a9e87db105604d8452e1/C, priority=11, startTime=1733324183827; duration=0sec 2024-12-04T14:56:24,716 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:24,716 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7618a4771c68a9e87db105604d8452e1:C 2024-12-04T14:56:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:24,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. as already flushing 2024-12-04T14:56:24,717 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-04T14:56:24,717 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:24,717 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:24,717 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:24,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:24,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741929_1105 (size=12301) 2024-12-04T14:56:24,726 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-04T14:56:24,726 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-04T14:56:24,726 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-04T14:56:24,726 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:24,726 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:24,726 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:24,726 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:24,726 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:24,726 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:24,726 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:24,726 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:24,726 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:24,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:24,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:24,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:24,736 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-04T14:56:24,736 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:24,736 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:24,736 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:24,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:25,124 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/eea75abc7d7d4f54a052e53c6c5503a1 2024-12-04T14:56:25,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/114cf1c83d764784bbeff74cb7663def as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/114cf1c83d764784bbeff74cb7663def 2024-12-04T14:56:25,138 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/114cf1c83d764784bbeff74cb7663def, entries=150, sequenceid=445, filesize=12.0 K 2024-12-04T14:56:25,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/80397ad544b54833bf330abed5536eb3 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/80397ad544b54833bf330abed5536eb3 2024-12-04T14:56:25,144 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/80397ad544b54833bf330abed5536eb3, entries=150, sequenceid=445, filesize=12.0 K 2024-12-04T14:56:25,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/eea75abc7d7d4f54a052e53c6c5503a1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/eea75abc7d7d4f54a052e53c6c5503a1 2024-12-04T14:56:25,150 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/eea75abc7d7d4f54a052e53c6c5503a1, entries=150, sequenceid=445, filesize=12.0 K 2024-12-04T14:56:25,151 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=33.54 KB/34350 for 7618a4771c68a9e87db105604d8452e1 in 1294ms, sequenceid=445, compaction requested=false 2024-12-04T14:56:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 7618a4771c68a9e87db105604d8452e1: 2024-12-04T14:56:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-04T14:56:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-04T14:56:25,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-04T14:56:25,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0630 sec 2024-12-04T14:56:25,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 2.0690 sec 2024-12-04T14:56:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T14:56:25,236 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:25,237 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T14:56:25,237 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-12-04T14:56:25,237 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-04T14:56:25,237 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-04T14:56:25,237 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-04T14:56:25,237 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-12-04T14:56:25,237 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T14:56:25,237 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4521 2024-12-04T14:56:25,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4373 2024-12-04T14:56:25,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T14:56:25,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2005 2024-12-04T14:56:25,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6014 rows 2024-12-04T14:56:25,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2028 2024-12-04T14:56:25,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6073 rows 2024-12-04T14:56:25,238 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:56:25,239 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:56:25,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:25,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:25,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:25,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T14:56:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T14:56:25,252 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324185252"}]},"ts":"1733324185252"} 2024-12-04T14:56:25,254 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T14:56:25,254 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T14:56:25,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:56:25,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7618a4771c68a9e87db105604d8452e1, UNASSIGN}] 2024-12-04T14:56:25,261 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7618a4771c68a9e87db105604d8452e1, UNASSIGN 2024-12-04T14:56:25,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=7618a4771c68a9e87db105604d8452e1, regionState=CLOSING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:25,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7618a4771c68a9e87db105604d8452e1, UNASSIGN because future has completed 2024-12-04T14:56:25,265 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:56:25,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:56:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T14:56:25,426 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:25,427 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:56:25,428 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 7618a4771c68a9e87db105604d8452e1, disabling compactions & flushes 2024-12-04T14:56:25,428 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:25,428 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:25,428 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. after waiting 0 ms 2024-12-04T14:56:25,428 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:25,428 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 7618a4771c68a9e87db105604d8452e1 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-04T14:56:25,433 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/8bd3fbdc8d6545289ada2c69f3c8cb62 is 50, key is test_row_0/A:col10/1733324184725/Put/seqid=0 2024-12-04T14:56:25,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741930_1106 (size=12301) 2024-12-04T14:56:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T14:56:25,839 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/8bd3fbdc8d6545289ada2c69f3c8cb62 2024-12-04T14:56:25,857 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/d8dd1716b9744c229c82fab63955d458 is 50, key is test_row_0/B:col10/1733324184725/Put/seqid=0 2024-12-04T14:56:25,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741931_1107 (size=12301) 2024-12-04T14:56:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T14:56:26,263 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/d8dd1716b9744c229c82fab63955d458 2024-12-04T14:56:26,277 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/4aea17b482de45b483f3c1528cae8380 is 50, key is test_row_0/C:col10/1733324184725/Put/seqid=0 2024-12-04T14:56:26,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741932_1108 (size=12301) 2024-12-04T14:56:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T14:56:26,576 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:56:26,683 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/4aea17b482de45b483f3c1528cae8380 2024-12-04T14:56:26,688 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/A/8bd3fbdc8d6545289ada2c69f3c8cb62 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/8bd3fbdc8d6545289ada2c69f3c8cb62 2024-12-04T14:56:26,692 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/8bd3fbdc8d6545289ada2c69f3c8cb62, entries=150, sequenceid=456, filesize=12.0 K 2024-12-04T14:56:26,693 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/B/d8dd1716b9744c229c82fab63955d458 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/d8dd1716b9744c229c82fab63955d458 2024-12-04T14:56:26,697 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/d8dd1716b9744c229c82fab63955d458, entries=150, sequenceid=456, filesize=12.0 K 2024-12-04T14:56:26,698 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/.tmp/C/4aea17b482de45b483f3c1528cae8380 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/4aea17b482de45b483f3c1528cae8380 2024-12-04T14:56:26,702 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/4aea17b482de45b483f3c1528cae8380, entries=150, sequenceid=456, filesize=12.0 K 2024-12-04T14:56:26,703 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 7618a4771c68a9e87db105604d8452e1 in 1274ms, sequenceid=456, compaction requested=true 2024-12-04T14:56:26,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ffd23fae4eb449290b156e8bdf157d9, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/64934dc1e8c241c9a344295e532a238e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7007a544dfec4d81937dc4b1a9466ae5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/a879a334b4b14c76aa3013161fdb519d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/79b20b2b66684f68aa972543fdb2b273, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f3dbe4e666fd424e9d2da4bb0c632bb8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/ee318cbb4fe74b179fa51e1321014913, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f1bd5c7a964b4a409fdf06baf38e05b6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/8edb7c3364b54c7ca992ff854bb56f82, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/eaa33088130740d399a8f3ce247790fc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/96e056fa022944f4a3f75f69deefe458, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2a0b6a81e382462c9eab44a85422e2d2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e3fb3e0fe5d04f528af7bf87d3519804, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7570e63617cf4dafbb587ec761c6d684, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2fd86e2ebdd244d6aada49137fa08274, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d09fd1714a8c4ed995c6f3411663ee58, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0b0b867c63fc4f95b29b44c6f82b3929, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/733a1407c4074855b3cde6c76eb3e7fa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d37627d5b93041a78827aeafb7af5731, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/529ae962b6404e4e868e08f9a10b1d41, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0d77f2e622bb47e2b33aca616539183f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/29b78613923245f79db8844a3b079f6f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1b278d8f4b684d12bca05076f3f51398, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1eae467861c84f3894702d04fbfb75fd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/dca78ba8ca8c44948e1390591d4e728a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ed00bd2d1fb4593b25cb1151fb00e53, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/25bac21c32a544969bc7f1087f0dadc1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2df62f4885e34f2c9786a7a3d159d99f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e0db9f6e0d7b451ab4d7ab4570308f1c] to archive 2024-12-04T14:56:26,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:56:26,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ffd23fae4eb449290b156e8bdf157d9 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ffd23fae4eb449290b156e8bdf157d9 2024-12-04T14:56:26,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/64934dc1e8c241c9a344295e532a238e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/64934dc1e8c241c9a344295e532a238e 2024-12-04T14:56:26,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7007a544dfec4d81937dc4b1a9466ae5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7007a544dfec4d81937dc4b1a9466ae5 2024-12-04T14:56:26,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/a879a334b4b14c76aa3013161fdb519d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/a879a334b4b14c76aa3013161fdb519d 2024-12-04T14:56:26,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/79b20b2b66684f68aa972543fdb2b273 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/79b20b2b66684f68aa972543fdb2b273 2024-12-04T14:56:26,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f3dbe4e666fd424e9d2da4bb0c632bb8 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f3dbe4e666fd424e9d2da4bb0c632bb8 2024-12-04T14:56:26,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/ee318cbb4fe74b179fa51e1321014913 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/ee318cbb4fe74b179fa51e1321014913 2024-12-04T14:56:26,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f1bd5c7a964b4a409fdf06baf38e05b6 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/f1bd5c7a964b4a409fdf06baf38e05b6 2024-12-04T14:56:26,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/8edb7c3364b54c7ca992ff854bb56f82 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/8edb7c3364b54c7ca992ff854bb56f82 2024-12-04T14:56:26,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/eaa33088130740d399a8f3ce247790fc to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/eaa33088130740d399a8f3ce247790fc 2024-12-04T14:56:26,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/96e056fa022944f4a3f75f69deefe458 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/96e056fa022944f4a3f75f69deefe458 2024-12-04T14:56:26,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2a0b6a81e382462c9eab44a85422e2d2 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2a0b6a81e382462c9eab44a85422e2d2 2024-12-04T14:56:26,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e3fb3e0fe5d04f528af7bf87d3519804 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e3fb3e0fe5d04f528af7bf87d3519804 2024-12-04T14:56:26,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7570e63617cf4dafbb587ec761c6d684 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/7570e63617cf4dafbb587ec761c6d684 2024-12-04T14:56:26,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2fd86e2ebdd244d6aada49137fa08274 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2fd86e2ebdd244d6aada49137fa08274 2024-12-04T14:56:26,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d09fd1714a8c4ed995c6f3411663ee58 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d09fd1714a8c4ed995c6f3411663ee58 2024-12-04T14:56:26,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0b0b867c63fc4f95b29b44c6f82b3929 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0b0b867c63fc4f95b29b44c6f82b3929 2024-12-04T14:56:26,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/733a1407c4074855b3cde6c76eb3e7fa to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/733a1407c4074855b3cde6c76eb3e7fa 2024-12-04T14:56:26,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d37627d5b93041a78827aeafb7af5731 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/d37627d5b93041a78827aeafb7af5731 2024-12-04T14:56:26,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/529ae962b6404e4e868e08f9a10b1d41 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/529ae962b6404e4e868e08f9a10b1d41 2024-12-04T14:56:26,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0d77f2e622bb47e2b33aca616539183f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/0d77f2e622bb47e2b33aca616539183f 2024-12-04T14:56:26,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/29b78613923245f79db8844a3b079f6f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/29b78613923245f79db8844a3b079f6f 2024-12-04T14:56:26,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1b278d8f4b684d12bca05076f3f51398 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1b278d8f4b684d12bca05076f3f51398 2024-12-04T14:56:26,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1eae467861c84f3894702d04fbfb75fd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/1eae467861c84f3894702d04fbfb75fd 2024-12-04T14:56:26,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/dca78ba8ca8c44948e1390591d4e728a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/dca78ba8ca8c44948e1390591d4e728a 2024-12-04T14:56:26,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ed00bd2d1fb4593b25cb1151fb00e53 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2ed00bd2d1fb4593b25cb1151fb00e53 2024-12-04T14:56:26,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/25bac21c32a544969bc7f1087f0dadc1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/25bac21c32a544969bc7f1087f0dadc1 2024-12-04T14:56:26,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2df62f4885e34f2c9786a7a3d159d99f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/2df62f4885e34f2c9786a7a3d159d99f 2024-12-04T14:56:26,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e0db9f6e0d7b451ab4d7ab4570308f1c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e0db9f6e0d7b451ab4d7ab4570308f1c 2024-12-04T14:56:26,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/89535a6c03bb400899f0f3c3c6ba707a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/510efa4a780645ab8502cc81e74216d1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b88bbcfc185c4008a079309c4bd77e10, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/663a359d3adf4ef29c7b7e7aeda7f60b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c69ff94f4c8d4068b33bf4221464339e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/baa5d4a467ac43bea31ee056543c9c3a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/9467ef5401134f1c8b21fa54068a893b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2710df85b43f460e8bff95a61a984614, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3671a6a3d51c42d2a9af00caf4e24e69, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/4a6b1adc492f4099992070b237ac7609, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/04b5eb7bdc8547d6aeb8b4cc428b33b5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/97d0f6baa87d4371bb86478af0feccca, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/f9c1159236fe4d8cb1dda5f5e5a4535c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/bb866eff2720498fbc4ad9f05a5bc227, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3ec72e3a6ccc4008b33fc2820a660cb5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c554074b600e414c87ff89cb1a595460, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2ea6de4f27144171ad2d50bba468de47, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/d1ee5cab13c642f3b5fc3b61c8bca6e1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c1ef008bc45d4e50b093ce3bf7ace4a5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b137e07adb0d44afa9b7f05bc97d191b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2e38a60d876d4430936e16f5bd112f8a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/25d9f3f6f2d743d79f2c9a1c1ce4c2ac, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/eef9c001ed894b3b8cdd00cb777c534c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/ee8b806511564e7b843baf9441816617, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/32a3a4e2045440ae8997bffcfb01cf7c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/be7c03813aa04252bdd414efb487c7c0, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/1afe723267994647989ea01d0ab18fdf, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/90853c57068f401286c4aac4b66db553, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/6a94600648754014a1be0c382b0d8e7f] to archive 2024-12-04T14:56:26,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:56:26,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/89535a6c03bb400899f0f3c3c6ba707a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/89535a6c03bb400899f0f3c3c6ba707a 2024-12-04T14:56:26,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/510efa4a780645ab8502cc81e74216d1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/510efa4a780645ab8502cc81e74216d1 2024-12-04T14:56:26,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b88bbcfc185c4008a079309c4bd77e10 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b88bbcfc185c4008a079309c4bd77e10 2024-12-04T14:56:26,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/663a359d3adf4ef29c7b7e7aeda7f60b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/663a359d3adf4ef29c7b7e7aeda7f60b 2024-12-04T14:56:26,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c69ff94f4c8d4068b33bf4221464339e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c69ff94f4c8d4068b33bf4221464339e 2024-12-04T14:56:26,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/baa5d4a467ac43bea31ee056543c9c3a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/baa5d4a467ac43bea31ee056543c9c3a 2024-12-04T14:56:26,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/9467ef5401134f1c8b21fa54068a893b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/9467ef5401134f1c8b21fa54068a893b 2024-12-04T14:56:26,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2710df85b43f460e8bff95a61a984614 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2710df85b43f460e8bff95a61a984614 2024-12-04T14:56:26,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3671a6a3d51c42d2a9af00caf4e24e69 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3671a6a3d51c42d2a9af00caf4e24e69 2024-12-04T14:56:26,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/4a6b1adc492f4099992070b237ac7609 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/4a6b1adc492f4099992070b237ac7609 2024-12-04T14:56:26,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/04b5eb7bdc8547d6aeb8b4cc428b33b5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/04b5eb7bdc8547d6aeb8b4cc428b33b5 2024-12-04T14:56:26,773 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/97d0f6baa87d4371bb86478af0feccca to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/97d0f6baa87d4371bb86478af0feccca 2024-12-04T14:56:26,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/f9c1159236fe4d8cb1dda5f5e5a4535c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/f9c1159236fe4d8cb1dda5f5e5a4535c 2024-12-04T14:56:26,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/bb866eff2720498fbc4ad9f05a5bc227 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/bb866eff2720498fbc4ad9f05a5bc227 2024-12-04T14:56:26,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3ec72e3a6ccc4008b33fc2820a660cb5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/3ec72e3a6ccc4008b33fc2820a660cb5 2024-12-04T14:56:26,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c554074b600e414c87ff89cb1a595460 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c554074b600e414c87ff89cb1a595460 2024-12-04T14:56:26,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2ea6de4f27144171ad2d50bba468de47 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2ea6de4f27144171ad2d50bba468de47 2024-12-04T14:56:26,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/d1ee5cab13c642f3b5fc3b61c8bca6e1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/d1ee5cab13c642f3b5fc3b61c8bca6e1 2024-12-04T14:56:26,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c1ef008bc45d4e50b093ce3bf7ace4a5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/c1ef008bc45d4e50b093ce3bf7ace4a5 2024-12-04T14:56:26,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b137e07adb0d44afa9b7f05bc97d191b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/b137e07adb0d44afa9b7f05bc97d191b 2024-12-04T14:56:26,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2e38a60d876d4430936e16f5bd112f8a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/2e38a60d876d4430936e16f5bd112f8a 2024-12-04T14:56:26,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/25d9f3f6f2d743d79f2c9a1c1ce4c2ac to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/25d9f3f6f2d743d79f2c9a1c1ce4c2ac 2024-12-04T14:56:26,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/eef9c001ed894b3b8cdd00cb777c534c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/eef9c001ed894b3b8cdd00cb777c534c 2024-12-04T14:56:26,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/ee8b806511564e7b843baf9441816617 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/ee8b806511564e7b843baf9441816617 2024-12-04T14:56:26,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/32a3a4e2045440ae8997bffcfb01cf7c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/32a3a4e2045440ae8997bffcfb01cf7c 2024-12-04T14:56:26,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/be7c03813aa04252bdd414efb487c7c0 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/be7c03813aa04252bdd414efb487c7c0 2024-12-04T14:56:26,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/1afe723267994647989ea01d0ab18fdf to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/1afe723267994647989ea01d0ab18fdf 2024-12-04T14:56:26,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/90853c57068f401286c4aac4b66db553 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/90853c57068f401286c4aac4b66db553 2024-12-04T14:56:26,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/6a94600648754014a1be0c382b0d8e7f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/6a94600648754014a1be0c382b0d8e7f 2024-12-04T14:56:26,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/c107dc89858743c5b451ade7443bab01, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8b05c525375f42e2a8d89e70fd3af429, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/1e814308ba1c4529b48a14c167c01b7f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bddc4df34e39423c801199a86fdcfbb1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8ef411078ad840a5aaf6196dd9ec8d12, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bde0990eb8714e2fb6a23291b6b22372, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/ad48e631386f4939a29d39c42b83d70d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/5734a2fc372b4641a89ddd406529c21c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/7750bdf426884f03abc3588bfdce1a3b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2a0e0755bfe44cf7932f35701e457cb1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/f379dbda303a481f9be805ff9cc787c1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/29850f880b264ab5b8bdbab6fc5b30aa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a672f41a261f45e9aae22f943e6f0fc1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/84ced081f1fc44d6b56e6ccd19400eea, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/65ddc623977947dcbb4ddbd4e0ef79fb, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/fea73fb269b24fc2a05fa433af389643, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/de9ae6b3bde44e03b9096374477fb9e5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/cac51c728bae4ff9ba15060f08572a03, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/52bb6e9bc5114fd4aed5d472a2e2b82a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/d80c4c7319b84098b4544532a8f9443c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/e145b3b41d6b422689245760f4a382de, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/9a15bb696487401caac65f0d85476f8c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/737f54b530764518b1a77f1e1c0bd65c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/00486d85780342d395b8ed54e1dd7e6f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/22f7cb8041ef4adb989b2d6f02ba291c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a195ee52fa014f8d8de99c308e0ef719, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2909e479e47148d9ac0157e59f596430, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/662746d035c14e9fbb024e5eb55a537b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/485eae8f0ad842209281e60b86b1fa04] to archive 2024-12-04T14:56:26,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:56:26,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/c107dc89858743c5b451ade7443bab01 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/c107dc89858743c5b451ade7443bab01 2024-12-04T14:56:26,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8b05c525375f42e2a8d89e70fd3af429 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8b05c525375f42e2a8d89e70fd3af429 2024-12-04T14:56:26,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/1e814308ba1c4529b48a14c167c01b7f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/1e814308ba1c4529b48a14c167c01b7f 2024-12-04T14:56:26,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bddc4df34e39423c801199a86fdcfbb1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bddc4df34e39423c801199a86fdcfbb1 2024-12-04T14:56:26,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8ef411078ad840a5aaf6196dd9ec8d12 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/8ef411078ad840a5aaf6196dd9ec8d12 2024-12-04T14:56:26,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bde0990eb8714e2fb6a23291b6b22372 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/bde0990eb8714e2fb6a23291b6b22372 2024-12-04T14:56:26,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/ad48e631386f4939a29d39c42b83d70d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/ad48e631386f4939a29d39c42b83d70d 2024-12-04T14:56:26,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/5734a2fc372b4641a89ddd406529c21c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/5734a2fc372b4641a89ddd406529c21c 2024-12-04T14:56:26,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/7750bdf426884f03abc3588bfdce1a3b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/7750bdf426884f03abc3588bfdce1a3b 2024-12-04T14:56:26,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2a0e0755bfe44cf7932f35701e457cb1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2a0e0755bfe44cf7932f35701e457cb1 2024-12-04T14:56:26,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/f379dbda303a481f9be805ff9cc787c1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/f379dbda303a481f9be805ff9cc787c1 2024-12-04T14:56:26,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/29850f880b264ab5b8bdbab6fc5b30aa to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/29850f880b264ab5b8bdbab6fc5b30aa 2024-12-04T14:56:26,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a672f41a261f45e9aae22f943e6f0fc1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a672f41a261f45e9aae22f943e6f0fc1 2024-12-04T14:56:26,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/84ced081f1fc44d6b56e6ccd19400eea to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/84ced081f1fc44d6b56e6ccd19400eea 2024-12-04T14:56:26,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/65ddc623977947dcbb4ddbd4e0ef79fb to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/65ddc623977947dcbb4ddbd4e0ef79fb 2024-12-04T14:56:26,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/fea73fb269b24fc2a05fa433af389643 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/fea73fb269b24fc2a05fa433af389643 2024-12-04T14:56:26,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/de9ae6b3bde44e03b9096374477fb9e5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/de9ae6b3bde44e03b9096374477fb9e5 2024-12-04T14:56:26,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/cac51c728bae4ff9ba15060f08572a03 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/cac51c728bae4ff9ba15060f08572a03 2024-12-04T14:56:26,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/52bb6e9bc5114fd4aed5d472a2e2b82a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/52bb6e9bc5114fd4aed5d472a2e2b82a 2024-12-04T14:56:26,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/d80c4c7319b84098b4544532a8f9443c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/d80c4c7319b84098b4544532a8f9443c 2024-12-04T14:56:26,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/e145b3b41d6b422689245760f4a382de to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/e145b3b41d6b422689245760f4a382de 2024-12-04T14:56:26,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/9a15bb696487401caac65f0d85476f8c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/9a15bb696487401caac65f0d85476f8c 2024-12-04T14:56:26,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/737f54b530764518b1a77f1e1c0bd65c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/737f54b530764518b1a77f1e1c0bd65c 2024-12-04T14:56:26,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/00486d85780342d395b8ed54e1dd7e6f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/00486d85780342d395b8ed54e1dd7e6f 2024-12-04T14:56:26,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/22f7cb8041ef4adb989b2d6f02ba291c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/22f7cb8041ef4adb989b2d6f02ba291c 2024-12-04T14:56:26,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a195ee52fa014f8d8de99c308e0ef719 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/a195ee52fa014f8d8de99c308e0ef719 2024-12-04T14:56:26,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2909e479e47148d9ac0157e59f596430 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/2909e479e47148d9ac0157e59f596430 2024-12-04T14:56:26,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/662746d035c14e9fbb024e5eb55a537b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/662746d035c14e9fbb024e5eb55a537b 2024-12-04T14:56:26,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/485eae8f0ad842209281e60b86b1fa04 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/485eae8f0ad842209281e60b86b1fa04 2024-12-04T14:56:26,827 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/recovered.edits/459.seqid, newMaxSeqId=459, maxSeqId=1 2024-12-04T14:56:26,828 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. 2024-12-04T14:56:26,828 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 7618a4771c68a9e87db105604d8452e1: Waiting for close lock at 1733324185428Running coprocessor pre-close hooks at 1733324185428Disabling compacts and flushes for region at 1733324185428Disabling writes for close at 1733324185428Obtaining lock to block concurrent updates at 1733324185428Preparing flush snapshotting stores in 7618a4771c68a9e87db105604d8452e1 at 1733324185428Finished memstore snapshotting TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733324185429 (+1 ms)Flushing stores of TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1. at 1733324185429Flushing 7618a4771c68a9e87db105604d8452e1/A: creating writer at 1733324185429Flushing 7618a4771c68a9e87db105604d8452e1/A: appending metadata at 1733324185433 (+4 ms)Flushing 7618a4771c68a9e87db105604d8452e1/A: closing flushed file at 1733324185433Flushing 7618a4771c68a9e87db105604d8452e1/B: creating writer at 1733324185850 (+417 ms)Flushing 7618a4771c68a9e87db105604d8452e1/B: appending metadata at 1733324185856 (+6 ms)Flushing 7618a4771c68a9e87db105604d8452e1/B: closing flushed file at 1733324185856Flushing 7618a4771c68a9e87db105604d8452e1/C: creating writer at 1733324186273 (+417 ms)Flushing 7618a4771c68a9e87db105604d8452e1/C: appending metadata at 1733324186276 (+3 ms)Flushing 7618a4771c68a9e87db105604d8452e1/C: closing flushed file at 1733324186277 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@139c5214: reopening flushed file at 1733324186687 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a1d8e92: reopening flushed file at 1733324186692 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a5828ed: reopening flushed file at 1733324186697 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 7618a4771c68a9e87db105604d8452e1 in 1274ms, sequenceid=456, compaction requested=true at 1733324186703 (+6 ms)Writing region close event to WAL at 1733324186822 (+119 ms)Running coprocessor post-close hooks at 1733324186828 (+6 ms)Closed at 1733324186828 2024-12-04T14:56:26,831 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:26,832 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=7618a4771c68a9e87db105604d8452e1, regionState=CLOSED 2024-12-04T14:56:26,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:56:26,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-04T14:56:26,839 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 7618a4771c68a9e87db105604d8452e1, server=d214edae7bd2,34477,1733324158096 in 1.5710 sec 2024-12-04T14:56:26,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-04T14:56:26,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7618a4771c68a9e87db105604d8452e1, UNASSIGN in 1.5790 sec 2024-12-04T14:56:26,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-04T14:56:26,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5860 sec 2024-12-04T14:56:26,846 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324186845"}]},"ts":"1733324186845"} 2024-12-04T14:56:26,847 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T14:56:26,847 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T14:56:26,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6000 sec 2024-12-04T14:56:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T14:56:27,407 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:27,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T14:56:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:27,417 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-04T14:56:27,419 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:27,423 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:27,428 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/recovered.edits] 2024-12-04T14:56:27,431 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/114cf1c83d764784bbeff74cb7663def to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/114cf1c83d764784bbeff74cb7663def 2024-12-04T14:56:27,432 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/8bd3fbdc8d6545289ada2c69f3c8cb62 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/8bd3fbdc8d6545289ada2c69f3c8cb62 2024-12-04T14:56:27,433 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e6ea2b2954cc429ca8584a46f0542c72 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/A/e6ea2b2954cc429ca8584a46f0542c72 2024-12-04T14:56:27,435 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/80397ad544b54833bf330abed5536eb3 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/80397ad544b54833bf330abed5536eb3 2024-12-04T14:56:27,436 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/d8dd1716b9744c229c82fab63955d458 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/d8dd1716b9744c229c82fab63955d458 2024-12-04T14:56:27,437 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/e344a551a4424b279cf0e3d5a247446a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/B/e344a551a4424b279cf0e3d5a247446a 2024-12-04T14:56:27,439 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/4aea17b482de45b483f3c1528cae8380 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/4aea17b482de45b483f3c1528cae8380 2024-12-04T14:56:27,440 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/9daf2d12f8f04a5e91a8d93df4218577 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/9daf2d12f8f04a5e91a8d93df4218577 2024-12-04T14:56:27,441 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/eea75abc7d7d4f54a052e53c6c5503a1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/C/eea75abc7d7d4f54a052e53c6c5503a1 2024-12-04T14:56:27,444 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/recovered.edits/459.seqid to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1/recovered.edits/459.seqid 2024-12-04T14:56:27,444 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/7618a4771c68a9e87db105604d8452e1 2024-12-04T14:56:27,444 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T14:56:27,449 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:27,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-04T14:56:27,456 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T14:56:27,483 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T14:56:27,485 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:27,485 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T14:56:27,485 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324187485"}]},"ts":"9223372036854775807"} 2024-12-04T14:56:27,490 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T14:56:27,490 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7618a4771c68a9e87db105604d8452e1, NAME => 'TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T14:56:27,490 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T14:56:27,490 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324187490"}]},"ts":"9223372036854775807"} 2024-12-04T14:56:27,493 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T14:56:27,494 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:27,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-12-04T14:56:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-04T14:56:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T14:56:27,526 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:27,536 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1963293955_22 at /127.0.0.1:39994 [Waiting for operation #229] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;d214edae7bd2:34477-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2070237321_22 at /127.0.0.1:40366 [Waiting for operation #144] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2070237321_22 at /127.0.0.1:40150 [Waiting for operation #223] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 199) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8996 (was 9530) 2024-12-04T14:56:27,544 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=8995 2024-12-04T14:56:27,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:56:27,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:56:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:27,548 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:56:27,548 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:27,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-12-04T14:56:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:56:27,549 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:56:27,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741933_1109 (size=959) 2024-12-04T14:56:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:56:27,750 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-04T14:56:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:56:27,959 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1 2024-12-04T14:56:27,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741934_1110 (size=53) 2024-12-04T14:56:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:56:28,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:56:28,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fc83efc20bfad49ed809f1d485751cce, disabling compactions & flushes 2024-12-04T14:56:28,368 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:28,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:28,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. after waiting 0 ms 2024-12-04T14:56:28,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:28,368 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:28,368 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fc83efc20bfad49ed809f1d485751cce: Waiting for close lock at 1733324188368Disabling compacts and flushes for region at 1733324188368Disabling writes for close at 1733324188368Writing region close event to WAL at 1733324188368Closed at 1733324188368 2024-12-04T14:56:28,369 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:56:28,370 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324188369"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324188369"}]},"ts":"1733324188369"} 2024-12-04T14:56:28,371 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T14:56:28,372 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:56:28,373 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324188372"}]},"ts":"1733324188372"} 2024-12-04T14:56:28,374 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T14:56:28,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, ASSIGN}] 2024-12-04T14:56:28,376 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, ASSIGN 2024-12-04T14:56:28,377 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, ASSIGN; state=OFFLINE, location=d214edae7bd2,34477,1733324158096; forceNewPlan=false, retain=false 2024-12-04T14:56:28,528 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=fc83efc20bfad49ed809f1d485751cce, regionState=OPENING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:28,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, ASSIGN because future has completed 2024-12-04T14:56:28,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:56:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:56:28,694 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:28,694 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:56:28,695 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,696 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:56:28,696 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,696 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,699 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,701 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc83efc20bfad49ed809f1d485751cce columnFamilyName A 2024-12-04T14:56:28,701 DEBUG [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:28,701 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(327): Store=fc83efc20bfad49ed809f1d485751cce/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:28,702 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,703 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc83efc20bfad49ed809f1d485751cce columnFamilyName B 2024-12-04T14:56:28,703 DEBUG [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:28,703 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(327): Store=fc83efc20bfad49ed809f1d485751cce/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:28,703 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,704 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc83efc20bfad49ed809f1d485751cce columnFamilyName C 2024-12-04T14:56:28,705 DEBUG [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:28,705 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(327): Store=fc83efc20bfad49ed809f1d485751cce/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:28,705 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,706 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,706 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,707 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,707 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,708 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:56:28,709 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,710 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:56:28,711 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened fc83efc20bfad49ed809f1d485751cce; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71099098, jitterRate=0.05945912003517151}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:56:28,711 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:28,712 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for fc83efc20bfad49ed809f1d485751cce: Running coprocessor pre-open hook at 1733324188697Writing region info on filesystem at 1733324188697Initializing all the Stores at 1733324188698 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324188698Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324188698Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324188698Cleaning up temporary data from old regions at 1733324188707 (+9 ms)Running coprocessor post-open hooks at 1733324188711 (+4 ms)Region opened successfully at 1733324188712 (+1 ms) 2024-12-04T14:56:28,712 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., pid=36, masterSystemTime=1733324188687 2024-12-04T14:56:28,714 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:28,714 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:28,715 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=fc83efc20bfad49ed809f1d485751cce, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:28,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:56:28,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-04T14:56:28,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 in 184 msec 2024-12-04T14:56:28,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-04T14:56:28,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, ASSIGN in 345 msec 2024-12-04T14:56:28,722 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:56:28,722 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324188722"}]},"ts":"1733324188722"} 2024-12-04T14:56:28,724 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T14:56:28,725 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:56:28,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-12-04T14:56:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T14:56:29,696 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:29,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:29,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:29,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:29,700 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@278dcaf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:29,704 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed0a68c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:29,706 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:29,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:29,712 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:56:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,45797,1733324157426 2024-12-04T14:56:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@d0ee34f 2024-12-04T14:56:29,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:56:29,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:56:29,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:56:29,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:56:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:29,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741935_1111 (size=995) 2024-12-04T14:56:30,141 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-04T14:56:30,141 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-04T14:56:30,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:56:30,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, REOPEN/MOVE}] 2024-12-04T14:56:30,158 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, REOPEN/MOVE 2024-12-04T14:56:30,158 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=fc83efc20bfad49ed809f1d485751cce, regionState=CLOSING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, REOPEN/MOVE because future has completed 2024-12-04T14:56:30,160 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:56:30,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:56:30,313 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,313 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:56:30,314 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing fc83efc20bfad49ed809f1d485751cce, disabling compactions & flushes 2024-12-04T14:56:30,314 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:30,314 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:30,314 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. after waiting 0 ms 2024-12-04T14:56:30,314 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:30,321 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T14:56:30,322 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:30,322 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for fc83efc20bfad49ed809f1d485751cce: Waiting for close lock at 1733324190313Running coprocessor pre-close hooks at 1733324190313Disabling compacts and flushes for region at 1733324190313Disabling writes for close at 1733324190314 (+1 ms)Writing region close event to WAL at 1733324190314Running coprocessor post-close hooks at 1733324190322 (+8 ms)Closed at 1733324190322 2024-12-04T14:56:30,323 WARN [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: fc83efc20bfad49ed809f1d485751cce to self. 2024-12-04T14:56:30,326 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,327 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=fc83efc20bfad49ed809f1d485751cce, regionState=CLOSED 2024-12-04T14:56:30,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:56:30,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-04T14:56:30,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 in 172 msec 2024-12-04T14:56:30,338 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, REOPEN/MOVE; state=CLOSED, location=d214edae7bd2,34477,1733324158096; forceNewPlan=false, retain=true 2024-12-04T14:56:30,489 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=fc83efc20bfad49ed809f1d485751cce, regionState=OPENING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, REOPEN/MOVE because future has completed 2024-12-04T14:56:30,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:56:30,658 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:30,658 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:56:30,658 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,659 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:56:30,659 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,659 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,664 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,669 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc83efc20bfad49ed809f1d485751cce columnFamilyName A 2024-12-04T14:56:30,671 DEBUG [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:30,672 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(327): Store=fc83efc20bfad49ed809f1d485751cce/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:30,673 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,673 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc83efc20bfad49ed809f1d485751cce columnFamilyName B 2024-12-04T14:56:30,673 DEBUG [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:30,674 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(327): Store=fc83efc20bfad49ed809f1d485751cce/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:30,674 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,675 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc83efc20bfad49ed809f1d485751cce columnFamilyName C 2024-12-04T14:56:30,675 DEBUG [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:30,676 INFO [StoreOpener-fc83efc20bfad49ed809f1d485751cce-1 {}] regionserver.HStore(327): Store=fc83efc20bfad49ed809f1d485751cce/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:30,676 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,677 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,678 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,679 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,679 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,680 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:56:30,681 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,682 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened fc83efc20bfad49ed809f1d485751cce; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59302046, jitterRate=-0.11633065342903137}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:56:30,682 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,684 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for fc83efc20bfad49ed809f1d485751cce: Running coprocessor pre-open hook at 1733324190659Writing region info on filesystem at 1733324190659Initializing all the Stores at 1733324190660 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324190660Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324190660Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324190661 (+1 ms)Cleaning up temporary data from old regions at 1733324190679 (+18 ms)Running coprocessor post-open hooks at 1733324190682 (+3 ms)Region opened successfully at 1733324190684 (+2 ms) 2024-12-04T14:56:30,684 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., pid=41, masterSystemTime=1733324190650 2024-12-04T14:56:30,686 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:30,686 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:30,687 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=fc83efc20bfad49ed809f1d485751cce, regionState=OPEN, openSeqNum=5, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:56:30,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-12-04T14:56:30,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 in 194 msec 2024-12-04T14:56:30,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-04T14:56:30,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, REOPEN/MOVE in 533 msec 2024-12-04T14:56:30,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-04T14:56:30,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 546 msec 2024-12-04T14:56:30,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 973 msec 2024-12-04T14:56:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-04T14:56:30,700 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:30,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:30,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:30,702 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622b9b0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:30,703 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799faa56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:30,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:30,705 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:30,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:30,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:30,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d758f02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:30,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:30,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:30,707 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2a27d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:30,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:30,709 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306a7d65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:30,711 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460d8399, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:30,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:30,713 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d66bdfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:30,714 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa8916, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:30,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:30,716 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d60c7db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:30,718 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e649119, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:30,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:30,720 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a0a9e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:30,721 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103c59f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:30,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:30,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d4a85e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:30,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26392fad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:30,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:30,725 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c5d935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:30,726 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e82b1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:30,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:30,728 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277b35e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:30,729 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364eb416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:30,731 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:30,731 DEBUG [Thread-579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:30,731 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:30,731 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:30,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:30,731 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:30,731 DEBUG [Thread-583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:30,732 DEBUG [Thread-579 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-04T14:56:30,732 DEBUG [Thread-581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:30,732 DEBUG [Thread-579 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:30,733 DEBUG [Thread-575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:30,734 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:30,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:30,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:30,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:56:30,734 DEBUG [Thread-583 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:30,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:30,735 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:30,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:30,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:30,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:30,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:30,738 DEBUG [Thread-577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:30,739 DEBUG [Thread-583 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:30,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:56:30,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:56:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:56:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:56:30,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:56:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:56:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:56:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:56:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:56:30,763 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:30,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:56:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324250805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324250805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324250808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:30,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324250810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:30,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041e8892263dea4b39a5cd884787dbccb7_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324190760/Put/seqid=0 2024-12-04T14:56:30,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324250812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:30,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741936_1112 (size=19474) 2024-12-04T14:56:30,837 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:30,844 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041e8892263dea4b39a5cd884787dbccb7_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041e8892263dea4b39a5cd884787dbccb7_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:56:30,846 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/3e440767e17e455b8800edded75b332d, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:30,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/3e440767e17e455b8800edded75b332d is 175, key is test_row_0/A:col10/1733324190760/Put/seqid=0 2024-12-04T14:56:30,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741937_1113 (size=56733) 2024-12-04T14:56:30,876 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/3e440767e17e455b8800edded75b332d 2024-12-04T14:56:30,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:30,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:30,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:30,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:30,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:30,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:30,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/8fd92e127023497bb65d402a07dd6261 is 50, key is test_row_0/B:col10/1733324190760/Put/seqid=0 2024-12-04T14:56:30,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324250925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324250926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:30,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324250926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:30,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324250926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:30,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324250926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:30,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741938_1114 (size=12001) 2024-12-04T14:56:30,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/8fd92e127023497bb65d402a07dd6261 2024-12-04T14:56:30,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/7bfd0bc5eb5d4de3b73891789105aa67 is 50, key is test_row_0/C:col10/1733324190760/Put/seqid=0 2024-12-04T14:56:30,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741939_1115 (size=12001) 2024-12-04T14:56:31,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:31,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:31,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:56:31,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324251136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324251136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324251136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324251136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324251145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:31,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:31,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:31,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:31,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:56:31,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/7bfd0bc5eb5d4de3b73891789105aa67 2024-12-04T14:56:31,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/3e440767e17e455b8800edded75b332d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/3e440767e17e455b8800edded75b332d 2024-12-04T14:56:31,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/3e440767e17e455b8800edded75b332d, entries=300, sequenceid=16, filesize=55.4 K 2024-12-04T14:56:31,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/8fd92e127023497bb65d402a07dd6261 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8fd92e127023497bb65d402a07dd6261 2024-12-04T14:56:31,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8fd92e127023497bb65d402a07dd6261, entries=150, sequenceid=16, filesize=11.7 K 2024-12-04T14:56:31,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/7bfd0bc5eb5d4de3b73891789105aa67 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7bfd0bc5eb5d4de3b73891789105aa67 2024-12-04T14:56:31,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7bfd0bc5eb5d4de3b73891789105aa67, entries=150, sequenceid=16, filesize=11.7 K 2024-12-04T14:56:31,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fc83efc20bfad49ed809f1d485751cce in 676ms, sequenceid=16, compaction requested=false 2024-12-04T14:56:31,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:31,449 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:31,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:56:31,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204335ce0baa72f4ec49122c7f866b784ef_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324190803/Put/seqid=0 2024-12-04T14:56:31,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324251459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324251459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324251462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324251461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324251473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741940_1116 (size=12154) 2024-12-04T14:56:31,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:31,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:31,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:31,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,512 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204335ce0baa72f4ec49122c7f866b784ef_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204335ce0baa72f4ec49122c7f866b784ef_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:31,513 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/cfb1786642a7493d8113938434fab250, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:31,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/cfb1786642a7493d8113938434fab250 is 175, key is test_row_0/A:col10/1733324190803/Put/seqid=0 2024-12-04T14:56:31,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741941_1117 (size=30955) 2024-12-04T14:56:31,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324251585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324251585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324251586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324251586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324251586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:31,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324251795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324251795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324251795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324251795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324251795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:31,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:31,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:31,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:56:31,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/cfb1786642a7493d8113938434fab250 2024-12-04T14:56:31,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/b2143e4963104f32be328989dc224659 is 50, key is test_row_0/B:col10/1733324190803/Put/seqid=0 2024-12-04T14:56:31,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741942_1118 (size=12001) 2024-12-04T14:56:31,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:31,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:31,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:31,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:31,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:32,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324252106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324252106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324252107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324252107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324252107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:32,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:32,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:32,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:32,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:32,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:32,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:32,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:32,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:32,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:32,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:32,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/b2143e4963104f32be328989dc224659 2024-12-04T14:56:32,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/7121f53c3fcd47d9a23152ab14e314c7 is 50, key is test_row_0/C:col10/1733324190803/Put/seqid=0 2024-12-04T14:56:32,358 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:56:32,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741943_1119 (size=12001) 2024-12-04T14:56:32,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/7121f53c3fcd47d9a23152ab14e314c7 2024-12-04T14:56:32,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/cfb1786642a7493d8113938434fab250 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/cfb1786642a7493d8113938434fab250 2024-12-04T14:56:32,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/cfb1786642a7493d8113938434fab250, entries=150, sequenceid=42, filesize=30.2 K 2024-12-04T14:56:32,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/b2143e4963104f32be328989dc224659 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/b2143e4963104f32be328989dc224659 2024-12-04T14:56:32,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/b2143e4963104f32be328989dc224659, entries=150, sequenceid=42, filesize=11.7 K 2024-12-04T14:56:32,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/7121f53c3fcd47d9a23152ab14e314c7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7121f53c3fcd47d9a23152ab14e314c7 2024-12-04T14:56:32,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7121f53c3fcd47d9a23152ab14e314c7, entries=150, sequenceid=42, filesize=11.7 K 2024-12-04T14:56:32,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for fc83efc20bfad49ed809f1d485751cce in 962ms, sequenceid=42, compaction requested=false 2024-12-04T14:56:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:32,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T14:56:32,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:32,433 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-04T14:56:32,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204edcb07c1f4b94e57a6da6d3963020858_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324191461/Put/seqid=0 2024-12-04T14:56:32,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741944_1120 (size=12154) 2024-12-04T14:56:32,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:32,492 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204edcb07c1f4b94e57a6da6d3963020858_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204edcb07c1f4b94e57a6da6d3963020858_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:32,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/b5a6c84665fa4626bc26df987dda24f6, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/b5a6c84665fa4626bc26df987dda24f6 is 175, key is test_row_0/A:col10/1733324191461/Put/seqid=0 2024-12-04T14:56:32,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741945_1121 (size=30955) 2024-12-04T14:56:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:32,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:32,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324252650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324252652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324252653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324252655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324252657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324252765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324252765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324252765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324252766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324252766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:56:32,908 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/b5a6c84665fa4626bc26df987dda24f6 2024-12-04T14:56:32,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/8dd74bc7ad784ffc9ae02827108beada is 50, key is test_row_0/B:col10/1733324191461/Put/seqid=0 2024-12-04T14:56:32,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741946_1122 (size=12001) 2024-12-04T14:56:32,944 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/8dd74bc7ad784ffc9ae02827108beada 2024-12-04T14:56:32,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/92de3235912c408eb9cfec07e4f3f945 is 50, key is test_row_0/C:col10/1733324191461/Put/seqid=0 2024-12-04T14:56:32,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741947_1123 (size=12001) 2024-12-04T14:56:32,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324252975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324252975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324252975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324252977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324252977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324253285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324253286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324253286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324253286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324253286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,373 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/92de3235912c408eb9cfec07e4f3f945 2024-12-04T14:56:33,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/b5a6c84665fa4626bc26df987dda24f6 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/b5a6c84665fa4626bc26df987dda24f6 2024-12-04T14:56:33,391 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/b5a6c84665fa4626bc26df987dda24f6, entries=150, sequenceid=52, filesize=30.2 K 2024-12-04T14:56:33,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/8dd74bc7ad784ffc9ae02827108beada as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8dd74bc7ad784ffc9ae02827108beada 2024-12-04T14:56:33,411 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8dd74bc7ad784ffc9ae02827108beada, entries=150, sequenceid=52, filesize=11.7 K 2024-12-04T14:56:33,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/92de3235912c408eb9cfec07e4f3f945 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/92de3235912c408eb9cfec07e4f3f945 2024-12-04T14:56:33,428 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/92de3235912c408eb9cfec07e4f3f945, entries=150, sequenceid=52, filesize=11.7 K 2024-12-04T14:56:33,429 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for fc83efc20bfad49ed809f1d485751cce in 996ms, sequenceid=52, compaction requested=true 2024-12-04T14:56:33,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:33,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:33,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-04T14:56:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-04T14:56:33,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-04T14:56:33,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6970 sec 2024-12-04T14:56:33,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.7040 sec 2024-12-04T14:56:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:33,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:33,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-12-04T14:56:33,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324253803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324253804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324253805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324253806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324253808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204669b0242116544f08ada05c406873ec0_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324193798/Put/seqid=0 2024-12-04T14:56:33,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741948_1124 (size=14594) 2024-12-04T14:56:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324253915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324253915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324253924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324253924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324254125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324254125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324254134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324254134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,236 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:34,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204669b0242116544f08ada05c406873ec0_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204669b0242116544f08ada05c406873ec0_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:34,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/dff2a2688e114961941b70be0c4c974c, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:34,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/dff2a2688e114961941b70be0c4c974c is 175, key is test_row_0/A:col10/1733324193798/Put/seqid=0 2024-12-04T14:56:34,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741949_1125 (size=39549) 2024-12-04T14:56:34,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324254435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324254435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324254445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324254445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,647 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/dff2a2688e114961941b70be0c4c974c 2024-12-04T14:56:34,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/8674f924d4004c06a107731c91a7eb6e is 50, key is test_row_0/B:col10/1733324193798/Put/seqid=0 2024-12-04T14:56:34,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741950_1126 (size=12001) 2024-12-04T14:56:34,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324254825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,826 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in fc83efc20bfad49ed809f1d485751cce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:56:34,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T14:56:34,895 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:34,895 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-04T14:56:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:56:34,897 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:34,898 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:34,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:34,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324254945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324254945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324254955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324254955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:56:35,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T14:56:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:35,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:35,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/8674f924d4004c06a107731c91a7eb6e 2024-12-04T14:56:35,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/bd38bf1be157498f89bb11ea0a80f2e7 is 50, key is test_row_0/C:col10/1733324193798/Put/seqid=0 2024-12-04T14:56:35,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741951_1127 (size=12001) 2024-12-04T14:56:35,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T14:56:35,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:35,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:56:35,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T14:56:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:35,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/bd38bf1be157498f89bb11ea0a80f2e7 2024-12-04T14:56:35,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/dff2a2688e114961941b70be0c4c974c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/dff2a2688e114961941b70be0c4c974c 2024-12-04T14:56:35,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/dff2a2688e114961941b70be0c4c974c, entries=200, sequenceid=82, filesize=38.6 K 2024-12-04T14:56:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/8674f924d4004c06a107731c91a7eb6e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8674f924d4004c06a107731c91a7eb6e 2024-12-04T14:56:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8674f924d4004c06a107731c91a7eb6e, entries=150, sequenceid=82, filesize=11.7 K 2024-12-04T14:56:35,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/bd38bf1be157498f89bb11ea0a80f2e7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/bd38bf1be157498f89bb11ea0a80f2e7 2024-12-04T14:56:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/bd38bf1be157498f89bb11ea0a80f2e7, entries=150, sequenceid=82, filesize=11.7 K 2024-12-04T14:56:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T14:56:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for fc83efc20bfad49ed809f1d485751cce in 1709ms, sequenceid=82, compaction requested=true 2024-12-04T14:56:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:35,509 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-12-04T14:56:35,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:35,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:35,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:35,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T14:56:35,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:35,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-04T14:56:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,510 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,510 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,516 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:35,517 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/A is initiating minor compaction (all files) 2024-12-04T14:56:35,517 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/A in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,517 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/3e440767e17e455b8800edded75b332d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/cfb1786642a7493d8113938434fab250, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/b5a6c84665fa4626bc26df987dda24f6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/dff2a2688e114961941b70be0c4c974c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=154.5 K 2024-12-04T14:56:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,517 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,517 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/3e440767e17e455b8800edded75b332d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/cfb1786642a7493d8113938434fab250, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/b5a6c84665fa4626bc26df987dda24f6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/dff2a2688e114961941b70be0c4c974c] 2024-12-04T14:56:35,518 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:35,518 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/C is initiating minor compaction (all files) 2024-12-04T14:56:35,518 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/C in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,518 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7bfd0bc5eb5d4de3b73891789105aa67, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7121f53c3fcd47d9a23152ab14e314c7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/92de3235912c408eb9cfec07e4f3f945, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/bd38bf1be157498f89bb11ea0a80f2e7] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=46.9 K 2024-12-04T14:56:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,519 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e440767e17e455b8800edded75b332d, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324190746 2024-12-04T14:56:35,519 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bfd0bc5eb5d4de3b73891789105aa67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324190746 2024-12-04T14:56:35,519 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfb1786642a7493d8113938434fab250, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733324190800 2024-12-04T14:56:35,520 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5a6c84665fa4626bc26df987dda24f6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733324191457 2024-12-04T14:56:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,520 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting dff2a2688e114961941b70be0c4c974c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733324192652 2024-12-04T14:56:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,521 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7121f53c3fcd47d9a23152ab14e314c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733324190800 2024-12-04T14:56:35,521 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 92de3235912c408eb9cfec07e4f3f945, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733324191457 2024-12-04T14:56:35,522 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting bd38bf1be157498f89bb11ea0a80f2e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733324192652 2024-12-04T14:56:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:56:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,556 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:35,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120475c31684da17434c96b38a128c1bd1e2_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_1/A:col10/1733324193804/Put/seqid=0 2024-12-04T14:56:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,560 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#C#compaction#109 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,560 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/620b7539bcb74e278a927f365fca7604 is 50, key is test_row_0/C:col10/1733324193798/Put/seqid=0 2024-12-04T14:56:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,563 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412048d1e2de780384870944a7ad7d0bf55e2_fc83efc20bfad49ed809f1d485751cce store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:35,566 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412048d1e2de780384870944a7ad7d0bf55e2_fc83efc20bfad49ed809f1d485751cce, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:35,567 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048d1e2de780384870944a7ad7d0bf55e2_fc83efc20bfad49ed809f1d485751cce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741953_1129 (size=12139) 2024-12-04T14:56:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741952_1128 (size=9714) 2024-12-04T14:56:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,609 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/620b7539bcb74e278a927f365fca7604 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/620b7539bcb74e278a927f365fca7604 2024-12-04T14:56:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741954_1130 (size=4469) 2024-12-04T14:56:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,612 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120475c31684da17434c96b38a128c1bd1e2_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120475c31684da17434c96b38a128c1bd1e2_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,614 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#A#compaction#110 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/54f29af579f44c95a7b6a7e6d4a079a7, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:35,614 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/9cb7c3e156f541748ee77c642e70e5b2 is 175, key is test_row_0/A:col10/1733324193798/Put/seqid=0 2024-12-04T14:56:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/54f29af579f44c95a7b6a7e6d4a079a7 is 175, key is test_row_1/A:col10/1733324193804/Put/seqid=0 2024-12-04T14:56:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,620 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/C of fc83efc20bfad49ed809f1d485751cce into 620b7539bcb74e278a927f365fca7604(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:35,620 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:35,620 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/C, priority=12, startTime=1733324195510; duration=0sec 2024-12-04T14:56:35,620 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:35,620 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:C 2024-12-04T14:56:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,620 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741955_1131 (size=31093) 2024-12-04T14:56:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,622 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:35,622 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/B is initiating minor compaction (all files) 2024-12-04T14:56:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,622 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/B in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:35,622 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8fd92e127023497bb65d402a07dd6261, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/b2143e4963104f32be328989dc224659, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8dd74bc7ad784ffc9ae02827108beada, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8674f924d4004c06a107731c91a7eb6e] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=46.9 K 2024-12-04T14:56:35,623 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd92e127023497bb65d402a07dd6261, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324190746 2024-12-04T14:56:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,624 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b2143e4963104f32be328989dc224659, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733324190800 2024-12-04T14:56:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,624 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dd74bc7ad784ffc9ae02827108beada, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733324191457 2024-12-04T14:56:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,625 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8674f924d4004c06a107731c91a7eb6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733324192652 2024-12-04T14:56:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741956_1132 (size=22361) 2024-12-04T14:56:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,628 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/54f29af579f44c95a7b6a7e6d4a079a7 2024-12-04T14:56:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,630 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/9cb7c3e156f541748ee77c642e70e5b2 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9cb7c3e156f541748ee77c642e70e5b2 2024-12-04T14:56:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,644 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/A of fc83efc20bfad49ed809f1d485751cce into 9cb7c3e156f541748ee77c642e70e5b2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,645 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:35,645 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/A, priority=12, startTime=1733324195509; duration=0sec 2024-12-04T14:56:35,645 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:35,645 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:A 2024-12-04T14:56:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/893bb734a27742e29be7085863686510 is 50, key is test_row_1/B:col10/1733324193804/Put/seqid=0 2024-12-04T14:56:35,647 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#B#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,648 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/4caf7b8565eb4e89904d9e07caab4939 is 50, key is test_row_0/B:col10/1733324193798/Put/seqid=0 2024-12-04T14:56:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741958_1134 (size=12139) 2024-12-04T14:56:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741957_1133 (size=9657) 2024-12-04T14:56:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,664 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/4caf7b8565eb4e89904d9e07caab4939 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/4caf7b8565eb4e89904d9e07caab4939 2024-12-04T14:56:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,672 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/B of fc83efc20bfad49ed809f1d485751cce into 4caf7b8565eb4e89904d9e07caab4939(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:35,672 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:35,672 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/B, priority=12, startTime=1733324195510; duration=0sec 2024-12-04T14:56:35,672 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:35,672 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:B 2024-12-04T14:56:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:56:36,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324256037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324256041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324256041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324256040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,059 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/893bb734a27742e29be7085863686510 2024-12-04T14:56:36,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/e5419f9ae55c44e4bc1f5297ba5f95d3 is 50, key is test_row_1/C:col10/1733324193804/Put/seqid=0 2024-12-04T14:56:36,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741959_1135 (size=9657) 2024-12-04T14:56:36,084 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/e5419f9ae55c44e4bc1f5297ba5f95d3 2024-12-04T14:56:36,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/54f29af579f44c95a7b6a7e6d4a079a7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/54f29af579f44c95a7b6a7e6d4a079a7 2024-12-04T14:56:36,105 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/54f29af579f44c95a7b6a7e6d4a079a7, entries=100, sequenceid=88, filesize=21.8 K 2024-12-04T14:56:36,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/893bb734a27742e29be7085863686510 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/893bb734a27742e29be7085863686510 2024-12-04T14:56:36,112 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/893bb734a27742e29be7085863686510, entries=100, sequenceid=88, filesize=9.4 K 2024-12-04T14:56:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/e5419f9ae55c44e4bc1f5297ba5f95d3 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e5419f9ae55c44e4bc1f5297ba5f95d3 2024-12-04T14:56:36,119 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e5419f9ae55c44e4bc1f5297ba5f95d3, entries=100, sequenceid=88, filesize=9.4 K 2024-12-04T14:56:36,120 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=194.56 KB/199230 for fc83efc20bfad49ed809f1d485751cce in 611ms, sequenceid=88, compaction requested=false 2024-12-04T14:56:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-04T14:56:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-04T14:56:36,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-04T14:56:36,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2240 sec 2024-12-04T14:56:36,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.2320 sec 2024-12-04T14:56:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:36,148 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:36,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-12-04T14:56:36,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324256149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324256157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324256157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324256157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046ecbdd04135044ae9eaab4469aedf654_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324196030/Put/seqid=0 2024-12-04T14:56:36,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741960_1136 (size=14594) 2024-12-04T14:56:36,179 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:36,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046ecbdd04135044ae9eaab4469aedf654_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046ecbdd04135044ae9eaab4469aedf654_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:36,186 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/9140204732dc4693840edd9640eba3e2, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:36,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/9140204732dc4693840edd9640eba3e2 is 175, key is test_row_0/A:col10/1733324196030/Put/seqid=0 2024-12-04T14:56:36,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741961_1137 (size=39549) 2024-12-04T14:56:36,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324256255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324256365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324256365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324256373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324256465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,603 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=124, memsize=67.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/9140204732dc4693840edd9640eba3e2 2024-12-04T14:56:36,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/c2f0e89cb1e04931bfc43f7f0c538829 is 50, key is test_row_0/B:col10/1733324196030/Put/seqid=0 2024-12-04T14:56:36,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741962_1138 (size=12001) 2024-12-04T14:56:36,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/c2f0e89cb1e04931bfc43f7f0c538829 2024-12-04T14:56:36,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/e358656bee7b4f9e8043f17f77e34a52 is 50, key is test_row_0/C:col10/1733324196030/Put/seqid=0 2024-12-04T14:56:36,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324256675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324256675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324256685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741963_1139 (size=12001) 2024-12-04T14:56:36,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324256775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:36,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324256835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:36,836 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in fc83efc20bfad49ed809f1d485751cce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:56:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T14:56:37,045 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:37,046 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-04T14:56:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T14:56:37,047 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:37,048 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:37,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:37,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/e358656bee7b4f9e8043f17f77e34a52 2024-12-04T14:56:37,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/9140204732dc4693840edd9640eba3e2 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9140204732dc4693840edd9640eba3e2 2024-12-04T14:56:37,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9140204732dc4693840edd9640eba3e2, entries=200, sequenceid=124, filesize=38.6 K 2024-12-04T14:56:37,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/c2f0e89cb1e04931bfc43f7f0c538829 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c2f0e89cb1e04931bfc43f7f0c538829 2024-12-04T14:56:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c2f0e89cb1e04931bfc43f7f0c538829, entries=150, sequenceid=124, filesize=11.7 K 2024-12-04T14:56:37,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/e358656bee7b4f9e8043f17f77e34a52 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e358656bee7b4f9e8043f17f77e34a52 2024-12-04T14:56:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e358656bee7b4f9e8043f17f77e34a52, entries=150, sequenceid=124, filesize=11.7 K 2024-12-04T14:56:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for fc83efc20bfad49ed809f1d485751cce in 1000ms, sequenceid=124, compaction requested=true 2024-12-04T14:56:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:37,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,148 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:37,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:37,149 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:37,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:37,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:37,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:37,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,150 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:37,150 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/B is initiating minor compaction (all files) 2024-12-04T14:56:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,150 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/B in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,150 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/4caf7b8565eb4e89904d9e07caab4939, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/893bb734a27742e29be7085863686510, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c2f0e89cb1e04931bfc43f7f0c538829] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=33.0 K 2024-12-04T14:56:37,150 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:37,150 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/A is initiating minor compaction (all files) 2024-12-04T14:56:37,150 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/A in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,150 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9cb7c3e156f541748ee77c642e70e5b2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/54f29af579f44c95a7b6a7e6d4a079a7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9140204732dc4693840edd9640eba3e2] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=90.8 K 2024-12-04T14:56:37,150 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,151 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9cb7c3e156f541748ee77c642e70e5b2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/54f29af579f44c95a7b6a7e6d4a079a7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9140204732dc4693840edd9640eba3e2] 2024-12-04T14:56:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,151 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 4caf7b8565eb4e89904d9e07caab4939, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733324192652 2024-12-04T14:56:37,151 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cb7c3e156f541748ee77c642e70e5b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733324192652 2024-12-04T14:56:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,152 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 893bb734a27742e29be7085863686510, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733324193804 2024-12-04T14:56:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,152 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f29af579f44c95a7b6a7e6d4a079a7, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733324193804 2024-12-04T14:56:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,152 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c2f0e89cb1e04931bfc43f7f0c538829, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733324196030 2024-12-04T14:56:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,152 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9140204732dc4693840edd9640eba3e2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733324196030 2024-12-04T14:56:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T14:56:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,163 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#B#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:37,164 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:37,164 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/ec37aef6aeae4165901207c7589b1be5 is 50, key is test_row_0/B:col10/1733324196030/Put/seqid=0 2024-12-04T14:56:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,171 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412042008a5ad0cb44b688f3242e77242a3f4_fc83efc20bfad49ed809f1d485751cce store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,173 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412042008a5ad0cb44b688f3242e77242a3f4_fc83efc20bfad49ed809f1d485751cce, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:37,173 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042008a5ad0cb44b688f3242e77242a3f4_fc83efc20bfad49ed809f1d485751cce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741964_1140 (size=12241) 2024-12-04T14:56:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,189 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/ec37aef6aeae4165901207c7589b1be5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ec37aef6aeae4165901207c7589b1be5 2024-12-04T14:56:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,196 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/B of fc83efc20bfad49ed809f1d485751cce into ec37aef6aeae4165901207c7589b1be5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:37,196 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:37,196 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/B, priority=13, startTime=1733324197148; duration=0sec 2024-12-04T14:56:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,196 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:37,196 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:B 2024-12-04T14:56:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,196 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,198 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:37,198 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/C is initiating minor compaction (all files) 2024-12-04T14:56:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,198 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/C in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:37,198 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/620b7539bcb74e278a927f365fca7604, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e5419f9ae55c44e4bc1f5297ba5f95d3, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e358656bee7b4f9e8043f17f77e34a52] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=33.0 K 2024-12-04T14:56:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,199 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 620b7539bcb74e278a927f365fca7604, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733324192652 2024-12-04T14:56:37,200 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e5419f9ae55c44e4bc1f5297ba5f95d3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733324193804 2024-12-04T14:56:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,200 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e358656bee7b4f9e8043f17f77e34a52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733324196030 2024-12-04T14:56:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T14:56:37,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:37,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:37,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:37,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-04T14:56:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-04T14:56:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-04T14:56:37,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-12-04T14:56:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 161 msec 2024-12-04T14:56:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741965_1141 (size=4469) 2024-12-04T14:56:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,231 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#C#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,232 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/f87d82de0c294e3cac44bee715558f31 is 50, key is test_row_0/C:col10/1733324196030/Put/seqid=0 2024-12-04T14:56:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741966_1142 (size=12241) 2024-12-04T14:56:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:37,321 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:37,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:56:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043d0a14b633a64eeca44a3e7220e17ab4_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324197320/Put/seqid=0 2024-12-04T14:56:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T14:56:37,366 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,367 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-04T14:56:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:56:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,373 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,374 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:37,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324257394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324257394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741968_1144 (size=24758) 2024-12-04T14:56:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324257396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324257399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,404 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,420 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043d0a14b633a64eeca44a3e7220e17ab4_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043d0a14b633a64eeca44a3e7220e17ab4_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:37,421 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/e31faf28402b4ab7923117634723d1e9, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:37,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/e31faf28402b4ab7923117634723d1e9 is 175, key is test_row_0/A:col10/1733324197320/Put/seqid=0 2024-12-04T14:56:37,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741967_1143 (size=74391) 2024-12-04T14:56:37,426 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/e31faf28402b4ab7923117634723d1e9 2024-12-04T14:56:37,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/e38306a45eaf4e39a6945d2a5d8a1acc is 50, key is test_row_0/B:col10/1733324197320/Put/seqid=0 2024-12-04T14:56:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:56:37,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741969_1145 (size=9757) 2024-12-04T14:56:37,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/e38306a45eaf4e39a6945d2a5d8a1acc 2024-12-04T14:56:37,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/7af0f2a7206a4d22a5321ce306059758 is 50, key is test_row_0/C:col10/1733324197320/Put/seqid=0 2024-12-04T14:56:37,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324257505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324257505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324257505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324257506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741970_1146 (size=9757) 2024-12-04T14:56:37,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/7af0f2a7206a4d22a5321ce306059758 2024-12-04T14:56:37,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:56:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:37,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:37,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/e31faf28402b4ab7923117634723d1e9 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e31faf28402b4ab7923117634723d1e9 2024-12-04T14:56:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:37,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e31faf28402b4ab7923117634723d1e9, entries=400, sequenceid=137, filesize=72.6 K 2024-12-04T14:56:37,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/e38306a45eaf4e39a6945d2a5d8a1acc as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/e38306a45eaf4e39a6945d2a5d8a1acc 2024-12-04T14:56:37,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/e38306a45eaf4e39a6945d2a5d8a1acc, entries=100, sequenceid=137, filesize=9.5 K 2024-12-04T14:56:37,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/7af0f2a7206a4d22a5321ce306059758 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7af0f2a7206a4d22a5321ce306059758 2024-12-04T14:56:37,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7af0f2a7206a4d22a5321ce306059758, entries=100, sequenceid=137, filesize=9.5 K 2024-12-04T14:56:37,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fc83efc20bfad49ed809f1d485751cce in 234ms, sequenceid=137, compaction requested=false 2024-12-04T14:56:37,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:37,622 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#A#compaction#118 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:37,623 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/d90bf07e5b1b43fa970a86a9ea31e5f2 is 175, key is test_row_0/A:col10/1733324196030/Put/seqid=0 2024-12-04T14:56:37,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741971_1147 (size=31195) 2024-12-04T14:56:37,676 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/d90bf07e5b1b43fa970a86a9ea31e5f2 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/d90bf07e5b1b43fa970a86a9ea31e5f2 2024-12-04T14:56:37,677 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/f87d82de0c294e3cac44bee715558f31 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f87d82de0c294e3cac44bee715558f31 2024-12-04T14:56:37,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T14:56:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:37,681 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:56:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:56:37,686 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/C of fc83efc20bfad49ed809f1d485751cce into f87d82de0c294e3cac44bee715558f31(size=12.0 K), total size for store is 21.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:37,686 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/A of fc83efc20bfad49ed809f1d485751cce into d90bf07e5b1b43fa970a86a9ea31e5f2(size=30.5 K), total size for store is 103.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:37,686 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:37,686 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:37,686 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/C, priority=13, startTime=1733324197149; duration=0sec 2024-12-04T14:56:37,686 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/A, priority=13, startTime=1733324197148; duration=0sec 2024-12-04T14:56:37,686 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:37,686 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:A 2024-12-04T14:56:37,688 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:37,689 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:C 2024-12-04T14:56:37,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120435db191c3e1348e8abe652bf4cc50e39_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324197394/Put/seqid=0 2024-12-04T14:56:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:37,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:37,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324257726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324257730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324257731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324257731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741972_1148 (size=12304) 2024-12-04T14:56:37,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:37,750 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-04T14:56:37,750 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-04T14:56:37,757 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120435db191c3e1348e8abe652bf4cc50e39_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120435db191c3e1348e8abe652bf4cc50e39_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:37,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/863c30e46d154eaab57b5208e6973f71, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:37,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/863c30e46d154eaab57b5208e6973f71 is 175, key is test_row_0/A:col10/1733324197394/Put/seqid=0 2024-12-04T14:56:37,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741973_1149 (size=31105) 2024-12-04T14:56:37,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324257835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324257844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324257844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324257845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:56:38,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324258044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324258055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324258055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324258055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,177 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/863c30e46d154eaab57b5208e6973f71 2024-12-04T14:56:38,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/473fb89f8fa443cab0a0d69bfa05a0ea is 50, key is test_row_0/B:col10/1733324197394/Put/seqid=0 2024-12-04T14:56:38,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741974_1150 (size=12151) 2024-12-04T14:56:38,193 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/473fb89f8fa443cab0a0d69bfa05a0ea 2024-12-04T14:56:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/035e317f2b2145aab30c72250f9d03ec is 50, key is test_row_0/C:col10/1733324197394/Put/seqid=0 2024-12-04T14:56:38,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741975_1151 (size=12151) 2024-12-04T14:56:38,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324258357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324258365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324258365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324258365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:56:38,610 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/035e317f2b2145aab30c72250f9d03ec 2024-12-04T14:56:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/863c30e46d154eaab57b5208e6973f71 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/863c30e46d154eaab57b5208e6973f71 2024-12-04T14:56:38,626 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/863c30e46d154eaab57b5208e6973f71, entries=150, sequenceid=162, filesize=30.4 K 2024-12-04T14:56:38,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/473fb89f8fa443cab0a0d69bfa05a0ea as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/473fb89f8fa443cab0a0d69bfa05a0ea 2024-12-04T14:56:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,634 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/473fb89f8fa443cab0a0d69bfa05a0ea, entries=150, sequenceid=162, filesize=11.9 K 2024-12-04T14:56:38,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/035e317f2b2145aab30c72250f9d03ec as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/035e317f2b2145aab30c72250f9d03ec 2024-12-04T14:56:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,642 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/035e317f2b2145aab30c72250f9d03ec, entries=150, sequenceid=162, filesize=11.9 K 2024-12-04T14:56:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,643 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fc83efc20bfad49ed809f1d485751cce in 962ms, sequenceid=162, compaction requested=true 2024-12-04T14:56:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:38,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-04T14:56:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-04T14:56:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-04T14:56:38,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2710 sec 2024-12-04T14:56:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.2810 sec 2024-12-04T14:56:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:38,869 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:38,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-04T14:56:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120478a7620443ef4d13a8b0829e741256fd_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324197729/Put/seqid=0 2024-12-04T14:56:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741976_1152 (size=12304) 2024-12-04T14:56:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,901 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,907 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120478a7620443ef4d13a8b0829e741256fd_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120478a7620443ef4d13a8b0829e741256fd_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,909 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/71f2fc52a5cd48eeaf654c96c0b5fcf9, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/71f2fc52a5cd48eeaf654c96c0b5fcf9 is 175, key is test_row_0/A:col10/1733324197729/Put/seqid=0 2024-12-04T14:56:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:38,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741977_1153 (size=31101) 2024-12-04T14:56:38,934 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/71f2fc52a5cd48eeaf654c96c0b5fcf9 2024-12-04T14:56:38,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324258924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324258926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324258932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324258935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:38,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/c9943a49c1514a3489fdfc801c18f7f9 is 50, key is test_row_0/B:col10/1733324197729/Put/seqid=0 2024-12-04T14:56:38,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741978_1154 (size=9757) 2024-12-04T14:56:38,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/c9943a49c1514a3489fdfc801c18f7f9 2024-12-04T14:56:38,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/c0f85c48400346ddb2981a53ba65cf28 is 50, key is test_row_0/C:col10/1733324197729/Put/seqid=0 2024-12-04T14:56:38,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741979_1155 (size=9757) 2024-12-04T14:56:39,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324259046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324259046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324259046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324259046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324259256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324259256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324259256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324259256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/c0f85c48400346ddb2981a53ba65cf28 2024-12-04T14:56:39,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/71f2fc52a5cd48eeaf654c96c0b5fcf9 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/71f2fc52a5cd48eeaf654c96c0b5fcf9 2024-12-04T14:56:39,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/71f2fc52a5cd48eeaf654c96c0b5fcf9, entries=150, sequenceid=176, filesize=30.4 K 2024-12-04T14:56:39,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/c9943a49c1514a3489fdfc801c18f7f9 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c9943a49c1514a3489fdfc801c18f7f9 2024-12-04T14:56:39,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c9943a49c1514a3489fdfc801c18f7f9, entries=100, sequenceid=176, filesize=9.5 K 2024-12-04T14:56:39,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/c0f85c48400346ddb2981a53ba65cf28 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/c0f85c48400346ddb2981a53ba65cf28 2024-12-04T14:56:39,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/c0f85c48400346ddb2981a53ba65cf28, entries=100, sequenceid=176, filesize=9.5 K 2024-12-04T14:56:39,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fc83efc20bfad49ed809f1d485751cce in 564ms, sequenceid=176, compaction requested=true 2024-12-04T14:56:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:39,434 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T14:56:39,434 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:39,436 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 167792 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:39,436 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43906 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:39,436 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/A is initiating minor compaction (all files) 2024-12-04T14:56:39,436 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/B is initiating minor compaction (all files) 2024-12-04T14:56:39,436 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/A in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:39,436 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/B in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:39,436 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/d90bf07e5b1b43fa970a86a9ea31e5f2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e31faf28402b4ab7923117634723d1e9, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/863c30e46d154eaab57b5208e6973f71, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/71f2fc52a5cd48eeaf654c96c0b5fcf9] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=163.9 K 2024-12-04T14:56:39,436 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ec37aef6aeae4165901207c7589b1be5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/e38306a45eaf4e39a6945d2a5d8a1acc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/473fb89f8fa443cab0a0d69bfa05a0ea, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c9943a49c1514a3489fdfc801c18f7f9] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=42.9 K 2024-12-04T14:56:39,436 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:39,436 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/d90bf07e5b1b43fa970a86a9ea31e5f2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e31faf28402b4ab7923117634723d1e9, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/863c30e46d154eaab57b5208e6973f71, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/71f2fc52a5cd48eeaf654c96c0b5fcf9] 2024-12-04T14:56:39,437 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting d90bf07e5b1b43fa970a86a9ea31e5f2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733324196030 2024-12-04T14:56:39,437 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting ec37aef6aeae4165901207c7589b1be5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733324196030 2024-12-04T14:56:39,437 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting e31faf28402b4ab7923117634723d1e9, keycount=400, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733324197266 2024-12-04T14:56:39,437 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e38306a45eaf4e39a6945d2a5d8a1acc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733324197276 2024-12-04T14:56:39,438 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 863c30e46d154eaab57b5208e6973f71, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733324197383 2024-12-04T14:56:39,438 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 473fb89f8fa443cab0a0d69bfa05a0ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733324197383 2024-12-04T14:56:39,438 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f2fc52a5cd48eeaf654c96c0b5fcf9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733324197729 2024-12-04T14:56:39,438 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c9943a49c1514a3489fdfc801c18f7f9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733324197729 2024-12-04T14:56:39,446 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:39,451 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#B#compaction#130 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:39,452 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/fd9cc5b1285f40928f97c7e86719a08e is 50, key is test_row_0/B:col10/1733324197729/Put/seqid=0 2024-12-04T14:56:39,469 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120472c9faabda1b427ea4a79b3949d70992_fc83efc20bfad49ed809f1d485751cce store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:39,472 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120472c9faabda1b427ea4a79b3949d70992_fc83efc20bfad49ed809f1d485751cce, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:39,472 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120472c9faabda1b427ea4a79b3949d70992_fc83efc20bfad49ed809f1d485751cce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:39,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741981_1157 (size=4469) 2024-12-04T14:56:39,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741980_1156 (size=12527) 2024-12-04T14:56:39,492 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/fd9cc5b1285f40928f97c7e86719a08e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fd9cc5b1285f40928f97c7e86719a08e 2024-12-04T14:56:39,498 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/B of fc83efc20bfad49ed809f1d485751cce into fd9cc5b1285f40928f97c7e86719a08e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:39,498 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:39,498 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/B, priority=12, startTime=1733324199434; duration=0sec 2024-12-04T14:56:39,498 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:39,498 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:B 2024-12-04T14:56:39,498 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:39,501 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43906 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:39,501 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/C is initiating minor compaction (all files) 2024-12-04T14:56:39,501 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/C in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:39,501 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f87d82de0c294e3cac44bee715558f31, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7af0f2a7206a4d22a5321ce306059758, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/035e317f2b2145aab30c72250f9d03ec, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/c0f85c48400346ddb2981a53ba65cf28] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=42.9 K 2024-12-04T14:56:39,502 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting f87d82de0c294e3cac44bee715558f31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733324196030 2024-12-04T14:56:39,502 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af0f2a7206a4d22a5321ce306059758, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733324197276 2024-12-04T14:56:39,503 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 035e317f2b2145aab30c72250f9d03ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733324197383 2024-12-04T14:56:39,503 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f85c48400346ddb2981a53ba65cf28, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733324197729 2024-12-04T14:56:39,511 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#C#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:39,512 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/cda103d6ebf547899524ed5690d48422 is 50, key is test_row_0/C:col10/1733324197729/Put/seqid=0 2024-12-04T14:56:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T14:56:39,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741982_1158 (size=12527) 2024-12-04T14:56:39,520 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:39,521 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-04T14:56:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:56:39,527 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:39,528 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:39,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:39,568 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:39,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:56:39,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324259582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324259586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324259586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324259587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047e5765f70c2a40f88fc03bdd1829c65e_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324198932/Put/seqid=0 2024-12-04T14:56:39,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741983_1159 (size=14794) 2024-12-04T14:56:39,634 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:56:39,642 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047e5765f70c2a40f88fc03bdd1829c65e_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047e5765f70c2a40f88fc03bdd1829c65e_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:39,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/0db854204aa44e15b01103040fbc5451, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:39,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/0db854204aa44e15b01103040fbc5451 is 175, key is test_row_0/A:col10/1733324198932/Put/seqid=0 2024-12-04T14:56:39,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741984_1160 (size=39749) 2024-12-04T14:56:39,666 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/0db854204aa44e15b01103040fbc5451 2024-12-04T14:56:39,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/ece534c299264333a2c5d1654c8d71de is 50, key is test_row_0/B:col10/1733324198932/Put/seqid=0 2024-12-04T14:56:39,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T14:56:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:39,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:39,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741985_1161 (size=12151) 2024-12-04T14:56:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324259695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324259695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324259695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324259695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T14:56:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:39,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:56:39,886 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#A#compaction#129 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:39,886 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/ae5a7732bf1546d7a324e4a39214dbdd is 175, key is test_row_0/A:col10/1733324197729/Put/seqid=0 2024-12-04T14:56:39,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741986_1162 (size=31588) 2024-12-04T14:56:39,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324259905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324259905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324259906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324259916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:39,924 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/cda103d6ebf547899524ed5690d48422 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cda103d6ebf547899524ed5690d48422 2024-12-04T14:56:39,932 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/C of fc83efc20bfad49ed809f1d485751cce into cda103d6ebf547899524ed5690d48422(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:39,932 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:39,932 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/C, priority=12, startTime=1733324199434; duration=0sec 2024-12-04T14:56:39,932 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:39,932 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:C 2024-12-04T14:56:39,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T14:56:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:39,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:39,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:40,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/ece534c299264333a2c5d1654c8d71de 2024-12-04T14:56:40,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/2d608a24c14c4e2cb1ee6d7fd7bb06e7 is 50, key is test_row_0/C:col10/1733324198932/Put/seqid=0 2024-12-04T14:56:40,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741987_1163 (size=12151) 2024-12-04T14:56:40,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T14:56:40,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:40,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:56:40,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324260215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324260215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324260215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324260225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T14:56:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:40,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:40,299 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/ae5a7732bf1546d7a324e4a39214dbdd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/ae5a7732bf1546d7a324e4a39214dbdd 2024-12-04T14:56:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:40,309 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/A of fc83efc20bfad49ed809f1d485751cce into ae5a7732bf1546d7a324e4a39214dbdd(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:40,309 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:40,309 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/A, priority=12, startTime=1733324199434; duration=0sec 2024-12-04T14:56:40,310 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:40,310 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:A 2024-12-04T14:56:40,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T14:56:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:40,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:40,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/2d608a24c14c4e2cb1ee6d7fd7bb06e7 2024-12-04T14:56:40,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/0db854204aa44e15b01103040fbc5451 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/0db854204aa44e15b01103040fbc5451 2024-12-04T14:56:40,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/0db854204aa44e15b01103040fbc5451, entries=200, sequenceid=202, filesize=38.8 K 2024-12-04T14:56:40,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/ece534c299264333a2c5d1654c8d71de as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ece534c299264333a2c5d1654c8d71de 2024-12-04T14:56:40,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ece534c299264333a2c5d1654c8d71de, entries=150, sequenceid=202, filesize=11.9 K 2024-12-04T14:56:40,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/2d608a24c14c4e2cb1ee6d7fd7bb06e7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/2d608a24c14c4e2cb1ee6d7fd7bb06e7 2024-12-04T14:56:40,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/2d608a24c14c4e2cb1ee6d7fd7bb06e7, entries=150, sequenceid=202, filesize=11.9 K 2024-12-04T14:56:40,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fc83efc20bfad49ed809f1d485751cce in 970ms, sequenceid=202, compaction requested=false 2024-12-04T14:56:40,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:40,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T14:56:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:40,604 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:56:40,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a773ecbfe36f4c13a8899a9055ad8e55_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324199581/Put/seqid=0 2024-12-04T14:56:40,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741988_1164 (size=12304) 2024-12-04T14:56:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:56:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:40,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324260753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324260753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324260755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324260755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54224 deadline: 1733324260865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324260865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324260865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324260865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324260865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,867 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in fc83efc20bfad49ed809f1d485751cce of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:56:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:41,022 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a773ecbfe36f4c13a8899a9055ad8e55_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a773ecbfe36f4c13a8899a9055ad8e55_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/1b3d91190c284f27bbd8af04d26a323c, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:41,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/1b3d91190c284f27bbd8af04d26a323c is 175, key is test_row_0/A:col10/1733324199581/Put/seqid=0 2024-12-04T14:56:41,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741989_1165 (size=31105) 2024-12-04T14:56:41,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324261075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324261075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324261075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324261075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,150 INFO [master/d214edae7bd2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T14:56:41,150 INFO [master/d214edae7bd2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T14:56:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324261385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324261385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324261385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324261385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,431 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/1b3d91190c284f27bbd8af04d26a323c 2024-12-04T14:56:41,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/607933b19e3e43b2bc708269c577c1fa is 50, key is test_row_0/B:col10/1733324199581/Put/seqid=0 2024-12-04T14:56:41,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741990_1166 (size=12151) 2024-12-04T14:56:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:56:41,844 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/607933b19e3e43b2bc708269c577c1fa 2024-12-04T14:56:41,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/198975da3b9546eb9a8edf6fa6282aa8 is 50, key is test_row_0/C:col10/1733324199581/Put/seqid=0 2024-12-04T14:56:41,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741991_1167 (size=12151) 2024-12-04T14:56:41,883 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/198975da3b9546eb9a8edf6fa6282aa8 2024-12-04T14:56:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/1b3d91190c284f27bbd8af04d26a323c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1b3d91190c284f27bbd8af04d26a323c 2024-12-04T14:56:41,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324261895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324261895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324261895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,898 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1b3d91190c284f27bbd8af04d26a323c, entries=150, sequenceid=215, filesize=30.4 K 2024-12-04T14:56:41,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324261895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:41,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:41,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/607933b19e3e43b2bc708269c577c1fa as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/607933b19e3e43b2bc708269c577c1fa 2024-12-04T14:56:41,907 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/607933b19e3e43b2bc708269c577c1fa, entries=150, sequenceid=215, filesize=11.9 K 2024-12-04T14:56:41,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/198975da3b9546eb9a8edf6fa6282aa8 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/198975da3b9546eb9a8edf6fa6282aa8 2024-12-04T14:56:41,914 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/198975da3b9546eb9a8edf6fa6282aa8, entries=150, sequenceid=215, filesize=11.9 K 2024-12-04T14:56:41,916 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fc83efc20bfad49ed809f1d485751cce in 1311ms, sequenceid=215, compaction requested=true 2024-12-04T14:56:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-04T14:56:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-04T14:56:41,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-04T14:56:41,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3900 sec 2024-12-04T14:56:41,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.4000 sec 2024-12-04T14:56:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:42,906 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:42,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-04T14:56:42,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040d7e5c4b463a4729a6a35a36a4acbb39_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324200753/Put/seqid=0 2024-12-04T14:56:42,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741992_1168 (size=14794) 2024-12-04T14:56:42,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324262917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:42,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324262918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:42,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:42,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:42,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:42,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324262918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:42,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324262919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324263025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324263025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324263025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324263025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324263235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324263235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324263235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324263235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,317 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:43,320 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040d7e5c4b463a4729a6a35a36a4acbb39_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040d7e5c4b463a4729a6a35a36a4acbb39_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:43,321 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/6be0425581654a67b0fbf79ea8712552, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:43,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/6be0425581654a67b0fbf79ea8712552 is 175, key is test_row_0/A:col10/1733324200753/Put/seqid=0 2024-12-04T14:56:43,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741993_1169 (size=39749) 2024-12-04T14:56:43,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324263545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324263545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324263545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324263545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:43,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:43,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T14:56:43,696 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:43,696 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-04T14:56:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T14:56:43,698 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:43,698 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:43,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:43,726 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/6be0425581654a67b0fbf79ea8712552 2024-12-04T14:56:43,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/22d87ab1304c4995a736d3414cc8d92f is 50, key is test_row_0/B:col10/1733324200753/Put/seqid=0 2024-12-04T14:56:43,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741994_1170 (size=12151) 2024-12-04T14:56:43,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/22d87ab1304c4995a736d3414cc8d92f 2024-12-04T14:56:43,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/8c895dc667b74cc2b41ddc8938e0359e is 50, key is test_row_0/C:col10/1733324200753/Put/seqid=0 2024-12-04T14:56:43,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741995_1171 (size=12151) 2024-12-04T14:56:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T14:56:43,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T14:56:43,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:43,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:43,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:43,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:43,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:44,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T14:56:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:44,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:44,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:44,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T14:56:44,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324264055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:44,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324264055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:44,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324264055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:44,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:44,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:44,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324264055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:44,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:44,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:44,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:44,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:44,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T14:56:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:44,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:44,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:44,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/8c895dc667b74cc2b41ddc8938e0359e 2024-12-04T14:56:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/6be0425581654a67b0fbf79ea8712552 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6be0425581654a67b0fbf79ea8712552 2024-12-04T14:56:44,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6be0425581654a67b0fbf79ea8712552, entries=200, sequenceid=241, filesize=38.8 K 2024-12-04T14:56:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/22d87ab1304c4995a736d3414cc8d92f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/22d87ab1304c4995a736d3414cc8d92f 2024-12-04T14:56:44,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/22d87ab1304c4995a736d3414cc8d92f, entries=150, sequenceid=241, filesize=11.9 K 2024-12-04T14:56:44,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/8c895dc667b74cc2b41ddc8938e0359e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8c895dc667b74cc2b41ddc8938e0359e 2024-12-04T14:56:44,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8c895dc667b74cc2b41ddc8938e0359e, entries=150, sequenceid=241, filesize=11.9 K 2024-12-04T14:56:44,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for fc83efc20bfad49ed809f1d485751cce in 1276ms, sequenceid=241, compaction requested=true 2024-12-04T14:56:44,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:44,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:44,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:44,183 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:44,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:44,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:44,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:44,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T14:56:44,183 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:44,184 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:44,184 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142191 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:44,184 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/A is initiating minor compaction (all files) 2024-12-04T14:56:44,184 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/B is initiating minor compaction (all files) 2024-12-04T14:56:44,184 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/A in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:44,184 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/ae5a7732bf1546d7a324e4a39214dbdd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/0db854204aa44e15b01103040fbc5451, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1b3d91190c284f27bbd8af04d26a323c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6be0425581654a67b0fbf79ea8712552] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=138.9 K 2024-12-04T14:56:44,184 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:44,184 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/ae5a7732bf1546d7a324e4a39214dbdd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/0db854204aa44e15b01103040fbc5451, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1b3d91190c284f27bbd8af04d26a323c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6be0425581654a67b0fbf79ea8712552] 2024-12-04T14:56:44,185 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/B in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:44,185 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fd9cc5b1285f40928f97c7e86719a08e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ece534c299264333a2c5d1654c8d71de, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/607933b19e3e43b2bc708269c577c1fa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/22d87ab1304c4995a736d3414cc8d92f] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=47.8 K 2024-12-04T14:56:44,185 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5a7732bf1546d7a324e4a39214dbdd, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733324197383 2024-12-04T14:56:44,185 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting fd9cc5b1285f40928f97c7e86719a08e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733324197383 2024-12-04T14:56:44,185 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0db854204aa44e15b01103040fbc5451, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733324198912 2024-12-04T14:56:44,185 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting ece534c299264333a2c5d1654c8d71de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733324198912 2024-12-04T14:56:44,186 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b3d91190c284f27bbd8af04d26a323c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733324199571 2024-12-04T14:56:44,186 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 607933b19e3e43b2bc708269c577c1fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733324199571 2024-12-04T14:56:44,186 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6be0425581654a67b0fbf79ea8712552, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733324200747 2024-12-04T14:56:44,186 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 22d87ab1304c4995a736d3414cc8d92f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733324200747 2024-12-04T14:56:44,194 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:44,194 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/73d10c1f154f4bb69e94987005ea8ce3 is 50, key is test_row_0/B:col10/1733324200753/Put/seqid=0 2024-12-04T14:56:44,195 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:44,199 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204a68624a8ceee46549a506551eaa6cde8_fc83efc20bfad49ed809f1d485751cce store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:44,201 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204a68624a8ceee46549a506551eaa6cde8_fc83efc20bfad49ed809f1d485751cce, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:44,202 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a68624a8ceee46549a506551eaa6cde8_fc83efc20bfad49ed809f1d485751cce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:44,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741996_1172 (size=12663) 2024-12-04T14:56:44,215 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/73d10c1f154f4bb69e94987005ea8ce3 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/73d10c1f154f4bb69e94987005ea8ce3 2024-12-04T14:56:44,220 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/B of fc83efc20bfad49ed809f1d485751cce into 73d10c1f154f4bb69e94987005ea8ce3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:44,220 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:44,220 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/B, priority=12, startTime=1733324204183; duration=0sec 2024-12-04T14:56:44,220 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:44,220 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:B 2024-12-04T14:56:44,220 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:56:44,221 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:56:44,222 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/C is initiating minor compaction (all files) 2024-12-04T14:56:44,222 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/C in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:44,222 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cda103d6ebf547899524ed5690d48422, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/2d608a24c14c4e2cb1ee6d7fd7bb06e7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/198975da3b9546eb9a8edf6fa6282aa8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8c895dc667b74cc2b41ddc8938e0359e] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=47.8 K 2024-12-04T14:56:44,222 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting cda103d6ebf547899524ed5690d48422, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733324197383 2024-12-04T14:56:44,223 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d608a24c14c4e2cb1ee6d7fd7bb06e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733324198912 2024-12-04T14:56:44,223 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 198975da3b9546eb9a8edf6fa6282aa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733324199571 2024-12-04T14:56:44,223 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c895dc667b74cc2b41ddc8938e0359e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733324200747 2024-12-04T14:56:44,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741997_1173 (size=4469) 2024-12-04T14:56:44,238 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#C#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:44,238 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/f0814bf70758435394542364cefa9fef is 50, key is test_row_0/C:col10/1733324200753/Put/seqid=0 2024-12-04T14:56:44,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741998_1174 (size=12663) 2024-12-04T14:56:44,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-04T14:56:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:44,310 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-04T14:56:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f95ba4d2b2d64e4b9e31af222c00961e_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324202918/Put/seqid=0 2024-12-04T14:56:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T14:56:44,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741999_1175 (size=12304) 2024-12-04T14:56:44,637 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:44,638 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/7dc2740c206242a0a0858808118e1c3e is 175, key is test_row_0/A:col10/1733324200753/Put/seqid=0 2024-12-04T14:56:44,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742000_1176 (size=31617) 2024-12-04T14:56:44,658 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/f0814bf70758435394542364cefa9fef as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f0814bf70758435394542364cefa9fef 2024-12-04T14:56:44,664 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/C of fc83efc20bfad49ed809f1d485751cce into f0814bf70758435394542364cefa9fef(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:44,664 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:44,664 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/C, priority=12, startTime=1733324204183; duration=0sec 2024-12-04T14:56:44,664 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:44,664 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:C 2024-12-04T14:56:44,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:44,732 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f95ba4d2b2d64e4b9e31af222c00961e_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f95ba4d2b2d64e4b9e31af222c00961e_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:44,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/1a2f88feea3e4b889318439eca65b64b, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:44,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/1a2f88feea3e4b889318439eca65b64b is 175, key is test_row_0/A:col10/1733324202918/Put/seqid=0 2024-12-04T14:56:44,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742001_1177 (size=31105) 2024-12-04T14:56:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T14:56:45,047 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/7dc2740c206242a0a0858808118e1c3e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/7dc2740c206242a0a0858808118e1c3e 2024-12-04T14:56:45,056 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/A of fc83efc20bfad49ed809f1d485751cce into 7dc2740c206242a0a0858808118e1c3e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:45,057 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:45,057 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/A, priority=12, startTime=1733324204182; duration=0sec 2024-12-04T14:56:45,057 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:45,057 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:A 2024-12-04T14:56:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:45,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:45,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324265090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324265090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324265091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324265091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,139 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/1a2f88feea3e4b889318439eca65b64b 2024-12-04T14:56:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/fdb6ae5283a2441b927ef8ebc3d95460 is 50, key is test_row_0/B:col10/1733324202918/Put/seqid=0 2024-12-04T14:56:45,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742002_1178 (size=12151) 2024-12-04T14:56:45,156 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/fdb6ae5283a2441b927ef8ebc3d95460 2024-12-04T14:56:45,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/cb7d82a8c34a465a9065b598d8fa3da6 is 50, key is test_row_0/C:col10/1733324202918/Put/seqid=0 2024-12-04T14:56:45,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742003_1179 (size=12151) 2024-12-04T14:56:45,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324265213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324265213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324265213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324265213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324265425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324265425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324265425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324265425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,571 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/cb7d82a8c34a465a9065b598d8fa3da6 2024-12-04T14:56:45,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/1a2f88feea3e4b889318439eca65b64b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1a2f88feea3e4b889318439eca65b64b 2024-12-04T14:56:45,580 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1a2f88feea3e4b889318439eca65b64b, entries=150, sequenceid=252, filesize=30.4 K 2024-12-04T14:56:45,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/fdb6ae5283a2441b927ef8ebc3d95460 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fdb6ae5283a2441b927ef8ebc3d95460 2024-12-04T14:56:45,585 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fdb6ae5283a2441b927ef8ebc3d95460, entries=150, sequenceid=252, filesize=11.9 K 2024-12-04T14:56:45,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-04T14:56:45,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/cb7d82a8c34a465a9065b598d8fa3da6 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cb7d82a8c34a465a9065b598d8fa3da6 2024-12-04T14:56:45,590 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cb7d82a8c34a465a9065b598d8fa3da6, entries=150, sequenceid=252, filesize=11.9 K 2024-12-04T14:56:45,591 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for fc83efc20bfad49ed809f1d485751cce in 1282ms, sequenceid=252, compaction requested=false 2024-12-04T14:56:45,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:45,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:45,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-04T14:56:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-04T14:56:45,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-04T14:56:45,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8950 sec 2024-12-04T14:56:45,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8990 sec 2024-12-04T14:56:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:45,737 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:45,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-04T14:56:45,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042b9bf28fd764419b91063a4fccc9ac5a_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324205737/Put/seqid=0 2024-12-04T14:56:45,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324265744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324265744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324265745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324265745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742004_1180 (size=17534) 2024-12-04T14:56:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T14:56:45,845 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:45,846 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-04T14:56:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T14:56:45,847 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:45,848 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:45,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:45,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324265855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324265855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324265856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324265856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T14:56:46,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324266065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324266065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324266065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324266065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:46,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,155 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042b9bf28fd764419b91063a4fccc9ac5a_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042b9bf28fd764419b91063a4fccc9ac5a_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:46,156 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/e3c4a4be4d0147808e85cfa8e62452af, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:46,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/e3c4a4be4d0147808e85cfa8e62452af is 175, key is test_row_0/A:col10/1733324205737/Put/seqid=0 2024-12-04T14:56:46,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742005_1181 (size=48639) 2024-12-04T14:56:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T14:56:46,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:46,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:46,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324266374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324266374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324266375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324266375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:46,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:46,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T14:56:46,562 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/e3c4a4be4d0147808e85cfa8e62452af 2024-12-04T14:56:46,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/1fd6d54a27f9495fa8d955ecf430a389 is 50, key is test_row_0/B:col10/1733324205737/Put/seqid=0 2024-12-04T14:56:46,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742006_1182 (size=12301) 2024-12-04T14:56:46,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:46,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:46,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324266885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324266885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324266885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324266885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:46,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:46,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:46,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:46,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T14:56:46,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/1fd6d54a27f9495fa8d955ecf430a389 2024-12-04T14:56:46,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/5de10f4cf3c34a3ca31708e3430fab97 is 50, key is test_row_0/C:col10/1733324205737/Put/seqid=0 2024-12-04T14:56:47,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742007_1183 (size=12301) 2024-12-04T14:56:47,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:47,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:47,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:47,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:47,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:47,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:47,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:47,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:47,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/5de10f4cf3c34a3ca31708e3430fab97 2024-12-04T14:56:47,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/e3c4a4be4d0147808e85cfa8e62452af as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e3c4a4be4d0147808e85cfa8e62452af 2024-12-04T14:56:47,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e3c4a4be4d0147808e85cfa8e62452af, entries=250, sequenceid=282, filesize=47.5 K 2024-12-04T14:56:47,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/1fd6d54a27f9495fa8d955ecf430a389 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1fd6d54a27f9495fa8d955ecf430a389 2024-12-04T14:56:47,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1fd6d54a27f9495fa8d955ecf430a389, entries=150, sequenceid=282, filesize=12.0 K 2024-12-04T14:56:47,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/5de10f4cf3c34a3ca31708e3430fab97 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/5de10f4cf3c34a3ca31708e3430fab97 2024-12-04T14:56:47,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/5de10f4cf3c34a3ca31708e3430fab97, entries=150, sequenceid=282, filesize=12.0 K 2024-12-04T14:56:47,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for fc83efc20bfad49ed809f1d485751cce in 1690ms, sequenceid=282, compaction requested=true 2024-12-04T14:56:47,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:47,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:47,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:47,427 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:47,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:47,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:47,427 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:47,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:47,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:47,430 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:47,430 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111361 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:47,430 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/A is initiating minor compaction (all files) 2024-12-04T14:56:47,430 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/B is initiating minor compaction (all files) 2024-12-04T14:56:47,430 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/B in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,430 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/A in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,430 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/73d10c1f154f4bb69e94987005ea8ce3, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fdb6ae5283a2441b927ef8ebc3d95460, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1fd6d54a27f9495fa8d955ecf430a389] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=36.2 K 2024-12-04T14:56:47,430 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/7dc2740c206242a0a0858808118e1c3e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1a2f88feea3e4b889318439eca65b64b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e3c4a4be4d0147808e85cfa8e62452af] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=108.8 K 2024-12-04T14:56:47,430 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,430 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/7dc2740c206242a0a0858808118e1c3e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1a2f88feea3e4b889318439eca65b64b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e3c4a4be4d0147808e85cfa8e62452af] 2024-12-04T14:56:47,430 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 73d10c1f154f4bb69e94987005ea8ce3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733324200747 2024-12-04T14:56:47,431 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting fdb6ae5283a2441b927ef8ebc3d95460, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733324202917 2024-12-04T14:56:47,431 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dc2740c206242a0a0858808118e1c3e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733324200747 2024-12-04T14:56:47,431 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fd6d54a27f9495fa8d955ecf430a389, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733324205090 2024-12-04T14:56:47,431 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a2f88feea3e4b889318439eca65b64b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733324202917 2024-12-04T14:56:47,432 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c4a4be4d0147808e85cfa8e62452af, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733324205085 2024-12-04T14:56:47,440 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:47,440 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:47,441 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/7f159cc9a57b4db68dc0f0e497a87fac is 50, key is test_row_0/B:col10/1733324205737/Put/seqid=0 2024-12-04T14:56:47,443 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412044c1eb6591a8e4b42b5163fcaae67df18_fc83efc20bfad49ed809f1d485751cce store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:47,445 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412044c1eb6591a8e4b42b5163fcaae67df18_fc83efc20bfad49ed809f1d485751cce, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:47,445 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044c1eb6591a8e4b42b5163fcaae67df18_fc83efc20bfad49ed809f1d485751cce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:47,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742008_1184 (size=12915) 2024-12-04T14:56:47,466 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/7f159cc9a57b4db68dc0f0e497a87fac as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/7f159cc9a57b4db68dc0f0e497a87fac 2024-12-04T14:56:47,472 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/B of fc83efc20bfad49ed809f1d485751cce into 7f159cc9a57b4db68dc0f0e497a87fac(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:47,472 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:47,472 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/B, priority=13, startTime=1733324207427; duration=0sec 2024-12-04T14:56:47,473 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:47,473 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:B 2024-12-04T14:56:47,473 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:47,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742009_1185 (size=4469) 2024-12-04T14:56:47,475 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:47,475 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/C is initiating minor compaction (all files) 2024-12-04T14:56:47,475 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#A#compaction#151 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:47,475 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/C in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,475 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f0814bf70758435394542364cefa9fef, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cb7d82a8c34a465a9065b598d8fa3da6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/5de10f4cf3c34a3ca31708e3430fab97] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=36.2 K 2024-12-04T14:56:47,476 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/373f0689bcaf40fab59610fe8b40bc3b is 175, key is test_row_0/A:col10/1733324205737/Put/seqid=0 2024-12-04T14:56:47,476 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting f0814bf70758435394542364cefa9fef, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733324200747 2024-12-04T14:56:47,477 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting cb7d82a8c34a465a9065b598d8fa3da6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733324202917 2024-12-04T14:56:47,478 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de10f4cf3c34a3ca31708e3430fab97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733324205090 2024-12-04T14:56:47,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742010_1186 (size=31869) 2024-12-04T14:56:47,488 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#C#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:47,489 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/01773999983842ada963f673a998f3ef is 50, key is test_row_0/C:col10/1733324205737/Put/seqid=0 2024-12-04T14:56:47,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742011_1187 (size=12915) 2024-12-04T14:56:47,513 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/01773999983842ada963f673a998f3ef as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/01773999983842ada963f673a998f3ef 2024-12-04T14:56:47,522 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/C of fc83efc20bfad49ed809f1d485751cce into 01773999983842ada963f673a998f3ef(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:47,522 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:47,522 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/C, priority=13, startTime=1733324207427; duration=0sec 2024-12-04T14:56:47,522 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:47,522 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:C 2024-12-04T14:56:47,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-04T14:56:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:47,528 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-04T14:56:47,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b733d20be7744e1695d5b53e8437dd50_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324205741/Put/seqid=0 2024-12-04T14:56:47,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742012_1188 (size=12454) 2024-12-04T14:56:47,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:47,547 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b733d20be7744e1695d5b53e8437dd50_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b733d20be7744e1695d5b53e8437dd50_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/29789b64cf8e4d74b85d74b58864ed01, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/29789b64cf8e4d74b85d74b58864ed01 is 175, key is test_row_0/A:col10/1733324205741/Put/seqid=0 2024-12-04T14:56:47,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742013_1189 (size=31255) 2024-12-04T14:56:47,898 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/373f0689bcaf40fab59610fe8b40bc3b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/373f0689bcaf40fab59610fe8b40bc3b 2024-12-04T14:56:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:47,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:47,903 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/A of fc83efc20bfad49ed809f1d485751cce into 373f0689bcaf40fab59610fe8b40bc3b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:47,903 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:47,903 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/A, priority=13, startTime=1733324207427; duration=0sec 2024-12-04T14:56:47,903 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:47,903 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:A 2024-12-04T14:56:47,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324267920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:47,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324267923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:47,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324267924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324267925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:47,962 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/29789b64cf8e4d74b85d74b58864ed01 2024-12-04T14:56:47,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/eafddb03c7e94acf94d84da577b0452b is 50, key is test_row_0/B:col10/1733324205741/Put/seqid=0 2024-12-04T14:56:47,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742014_1190 (size=12301) 2024-12-04T14:56:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T14:56:48,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324268035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324268035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324268035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324268035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324268245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324268245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324268245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324268245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,373 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/eafddb03c7e94acf94d84da577b0452b 2024-12-04T14:56:48,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/8619f02f73ea4c479828068ab78cff1e is 50, key is test_row_0/C:col10/1733324205741/Put/seqid=0 2024-12-04T14:56:48,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742015_1191 (size=12301) 2024-12-04T14:56:48,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324268555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324268555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324268555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:48,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324268555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:48,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:48,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:48,785 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/8619f02f73ea4c479828068ab78cff1e 2024-12-04T14:56:48,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/29789b64cf8e4d74b85d74b58864ed01 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/29789b64cf8e4d74b85d74b58864ed01 2024-12-04T14:56:48,793 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/29789b64cf8e4d74b85d74b58864ed01, entries=150, sequenceid=293, filesize=30.5 K 2024-12-04T14:56:48,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/eafddb03c7e94acf94d84da577b0452b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/eafddb03c7e94acf94d84da577b0452b 2024-12-04T14:56:48,798 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/eafddb03c7e94acf94d84da577b0452b, entries=150, sequenceid=293, filesize=12.0 K 2024-12-04T14:56:48,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/8619f02f73ea4c479828068ab78cff1e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8619f02f73ea4c479828068ab78cff1e 2024-12-04T14:56:48,804 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8619f02f73ea4c479828068ab78cff1e, entries=150, sequenceid=293, filesize=12.0 K 2024-12-04T14:56:48,804 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for fc83efc20bfad49ed809f1d485751cce in 1277ms, sequenceid=293, compaction requested=false 2024-12-04T14:56:48,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:48,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:48,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-04T14:56:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-04T14:56:48,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-04T14:56:48,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9580 sec 2024-12-04T14:56:48,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.9620 sec 2024-12-04T14:56:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:49,066 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:49,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-12-04T14:56:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324269070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120473c220397b314ba88b4364d088eddf54_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324207924/Put/seqid=0 2024-12-04T14:56:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324269072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324269072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324269073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742016_1192 (size=14994) 2024-12-04T14:56:49,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324269175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324269185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324269185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324269185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324269385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324269395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324269395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324269395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:49,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120473c220397b314ba88b4364d088eddf54_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120473c220397b314ba88b4364d088eddf54_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:49,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/146137fad8414b399e051d531873baa7, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:49,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/146137fad8414b399e051d531873baa7 is 175, key is test_row_0/A:col10/1733324207924/Put/seqid=0 2024-12-04T14:56:49,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742017_1193 (size=39949) 2024-12-04T14:56:49,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324269701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324269705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324269705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324269705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:49,893 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/146137fad8414b399e051d531873baa7 2024-12-04T14:56:49,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/1a78cbb315974ddea7aa831287d37c4d is 50, key is test_row_0/B:col10/1733324207924/Put/seqid=0 2024-12-04T14:56:49,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742018_1194 (size=12301) 2024-12-04T14:56:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T14:56:50,005 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:50,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-04T14:56:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T14:56:50,007 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:50,007 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:50,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T14:56:50,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-04T14:56:50,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:50,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:50,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:50,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54234 deadline: 1733324270215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:50,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54244 deadline: 1733324270215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:50,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:50,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:50,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54232 deadline: 1733324270215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:50,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54168 deadline: 1733324270215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:56:50,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/1a78cbb315974ddea7aa831287d37c4d 2024-12-04T14:56:50,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-04T14:56:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/d30a85db3431402c9caaa899b74b9e9c is 50, key is test_row_0/C:col10/1733324207924/Put/seqid=0 2024-12-04T14:56:50,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742019_1195 (size=12301) 2024-12-04T14:56:50,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T14:56:50,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-04T14:56:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-04T14:56:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T14:56:50,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/d30a85db3431402c9caaa899b74b9e9c 2024-12-04T14:56:50,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/146137fad8414b399e051d531873baa7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/146137fad8414b399e051d531873baa7 2024-12-04T14:56:50,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/146137fad8414b399e051d531873baa7, entries=200, sequenceid=322, filesize=39.0 K 2024-12-04T14:56:50,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/1a78cbb315974ddea7aa831287d37c4d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1a78cbb315974ddea7aa831287d37c4d 2024-12-04T14:56:50,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1a78cbb315974ddea7aa831287d37c4d, entries=150, sequenceid=322, filesize=12.0 K 2024-12-04T14:56:50,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/d30a85db3431402c9caaa899b74b9e9c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/d30a85db3431402c9caaa899b74b9e9c 2024-12-04T14:56:50,733 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-12-04T14:56:50,733 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:50,733 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:50,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:50,733 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:50,734 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-12-04T14:56:50,734 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:50,734 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:50,735 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:50,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:50,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/d30a85db3431402c9caaa899b74b9e9c, entries=150, sequenceid=322, filesize=12.0 K 2024-12-04T14:56:50,736 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-12-04T14:56:50,737 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:50,737 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:50,737 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:50,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:50,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for fc83efc20bfad49ed809f1d485751cce in 1671ms, sequenceid=322, compaction requested=true 2024-12-04T14:56:50,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:50,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:56:50,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:50,737 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:50,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:56:50,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:50,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fc83efc20bfad49ed809f1d485751cce:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:56:50,737 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:50,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:50,738 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-12-04T14:56:50,738 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:50,738 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:50,738 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:50,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:50,738 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:50,738 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:50,738 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/B is initiating minor compaction (all files) 2024-12-04T14:56:50,738 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/A is initiating minor compaction (all files) 2024-12-04T14:56:50,739 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/B in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,739 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/A in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,739 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/373f0689bcaf40fab59610fe8b40bc3b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/29789b64cf8e4d74b85d74b58864ed01, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/146137fad8414b399e051d531873baa7] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=100.7 K 2024-12-04T14:56:50,739 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/7f159cc9a57b4db68dc0f0e497a87fac, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/eafddb03c7e94acf94d84da577b0452b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1a78cbb315974ddea7aa831287d37c4d] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=36.6 K 2024-12-04T14:56:50,739 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,739 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/373f0689bcaf40fab59610fe8b40bc3b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/29789b64cf8e4d74b85d74b58864ed01, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/146137fad8414b399e051d531873baa7] 2024-12-04T14:56:50,739 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f159cc9a57b4db68dc0f0e497a87fac, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733324205090 2024-12-04T14:56:50,739 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 373f0689bcaf40fab59610fe8b40bc3b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733324205090 2024-12-04T14:56:50,739 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting eafddb03c7e94acf94d84da577b0452b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733324205741 2024-12-04T14:56:50,739 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29789b64cf8e4d74b85d74b58864ed01, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733324205741 2024-12-04T14:56:50,739 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a78cbb315974ddea7aa831287d37c4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733324207922 2024-12-04T14:56:50,739 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 146137fad8414b399e051d531873baa7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733324207917 2024-12-04T14:56:50,744 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:50,745 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#B#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:50,745 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/f331ea2634294b28ad73602ad51bd9c6 is 50, key is test_row_0/B:col10/1733324207924/Put/seqid=0 2024-12-04T14:56:50,746 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204f11fe18d94fc48cd8c4933adb94f9cad_fc83efc20bfad49ed809f1d485751cce store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:50,747 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204f11fe18d94fc48cd8c4933adb94f9cad_fc83efc20bfad49ed809f1d485751cce, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:50,747 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f11fe18d94fc48cd8c4933adb94f9cad_fc83efc20bfad49ed809f1d485751cce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:50,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742020_1196 (size=13017) 2024-12-04T14:56:50,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742021_1197 (size=4469) 2024-12-04T14:56:50,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-04T14:56:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:50,772 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-04T14:56:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045fb1cfa8aa394e1aa8c97d6adf55acf2_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324209069/Put/seqid=0 2024-12-04T14:56:50,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742022_1198 (size=12454) 2024-12-04T14:56:50,909 INFO [Thread-581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-581. 2024-12-04T14:56:50,909 DEBUG [Thread-581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:50,909 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:50,910 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:50,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T14:56:51,153 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#A#compaction#160 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:51,154 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/15c30274e8ac4677bdd442f09a2d07e5 is 175, key is test_row_0/A:col10/1733324207924/Put/seqid=0 2024-12-04T14:56:51,159 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/f331ea2634294b28ad73602ad51bd9c6 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/f331ea2634294b28ad73602ad51bd9c6 2024-12-04T14:56:51,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742023_1199 (size=31971) 2024-12-04T14:56:51,165 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/B of fc83efc20bfad49ed809f1d485751cce into f331ea2634294b28ad73602ad51bd9c6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:51,165 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:51,165 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/B, priority=13, startTime=1733324210737; duration=0sec 2024-12-04T14:56:51,165 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:56:51,165 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:B 2024-12-04T14:56:51,165 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:56:51,166 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:56:51,166 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): fc83efc20bfad49ed809f1d485751cce/C is initiating minor compaction (all files) 2024-12-04T14:56:51,166 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fc83efc20bfad49ed809f1d485751cce/C in TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:51,167 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/01773999983842ada963f673a998f3ef, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8619f02f73ea4c479828068ab78cff1e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/d30a85db3431402c9caaa899b74b9e9c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp, totalSize=36.6 K 2024-12-04T14:56:51,167 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 01773999983842ada963f673a998f3ef, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733324205090 2024-12-04T14:56:51,167 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8619f02f73ea4c479828068ab78cff1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733324205741 2024-12-04T14:56:51,168 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting d30a85db3431402c9caaa899b74b9e9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733324207922 2024-12-04T14:56:51,174 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fc83efc20bfad49ed809f1d485751cce#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:56:51,174 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/f93eab560dd74463a764073ed0dbbb13 is 50, key is test_row_0/C:col10/1733324207924/Put/seqid=0 2024-12-04T14:56:51,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742024_1200 (size=13017) 2024-12-04T14:56:51,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:51,187 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045fb1cfa8aa394e1aa8c97d6adf55acf2_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045fb1cfa8aa394e1aa8c97d6adf55acf2_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:51,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/babc80e0afbe471590b4da8ee7edb59e, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:51,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/babc80e0afbe471590b4da8ee7edb59e is 175, key is test_row_0/A:col10/1733324209069/Put/seqid=0 2024-12-04T14:56:51,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742025_1201 (size=31255) 2024-12-04T14:56:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:51,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. as already flushing 2024-12-04T14:56:51,226 INFO [Thread-583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-583. 2024-12-04T14:56:51,227 DEBUG [Thread-583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:51,227 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:51,227 INFO [Thread-575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-575. 2024-12-04T14:56:51,227 INFO [Thread-577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-577. 2024-12-04T14:56:51,227 DEBUG [Thread-575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:51,227 DEBUG [Thread-577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:51,227 DEBUG [Thread-575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:51,227 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:51,227 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:51,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:51,227 DEBUG [Thread-575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:51,227 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:51,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:51,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:51,228 INFO [Thread-579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-579. 2024-12-04T14:56:51,228 DEBUG [Thread-579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:56:51,228 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:51,228 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:51,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:51,574 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/15c30274e8ac4677bdd442f09a2d07e5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/15c30274e8ac4677bdd442f09a2d07e5 2024-12-04T14:56:51,581 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/A of fc83efc20bfad49ed809f1d485751cce into 15c30274e8ac4677bdd442f09a2d07e5(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:51,581 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:51,581 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/A, priority=13, startTime=1733324210737; duration=0sec 2024-12-04T14:56:51,581 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:51,581 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:A 2024-12-04T14:56:51,584 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/f93eab560dd74463a764073ed0dbbb13 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f93eab560dd74463a764073ed0dbbb13 2024-12-04T14:56:51,591 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fc83efc20bfad49ed809f1d485751cce/C of fc83efc20bfad49ed809f1d485751cce into f93eab560dd74463a764073ed0dbbb13(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:56:51,591 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:51,591 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., storeName=fc83efc20bfad49ed809f1d485751cce/C, priority=13, startTime=1733324210737; duration=0sec 2024-12-04T14:56:51,591 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:56:51,591 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fc83efc20bfad49ed809f1d485751cce:C 2024-12-04T14:56:51,593 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/babc80e0afbe471590b4da8ee7edb59e 2024-12-04T14:56:51,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/d8b16ea9dac04d1a912999606d032cfb is 50, key is test_row_0/B:col10/1733324209069/Put/seqid=0 2024-12-04T14:56:51,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742026_1202 (size=12301) 2024-12-04T14:56:52,042 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/d8b16ea9dac04d1a912999606d032cfb 2024-12-04T14:56:52,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/00180ac12e6941419c30642a2322014a is 50, key is test_row_0/C:col10/1733324209069/Put/seqid=0 2024-12-04T14:56:52,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742027_1203 (size=12301) 2024-12-04T14:56:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T14:56:52,459 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/00180ac12e6941419c30642a2322014a 2024-12-04T14:56:52,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/babc80e0afbe471590b4da8ee7edb59e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/babc80e0afbe471590b4da8ee7edb59e 2024-12-04T14:56:52,471 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/babc80e0afbe471590b4da8ee7edb59e, entries=150, sequenceid=331, filesize=30.5 K 2024-12-04T14:56:52,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/d8b16ea9dac04d1a912999606d032cfb as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/d8b16ea9dac04d1a912999606d032cfb 2024-12-04T14:56:52,476 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/d8b16ea9dac04d1a912999606d032cfb, entries=150, sequenceid=331, filesize=12.0 K 2024-12-04T14:56:52,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/00180ac12e6941419c30642a2322014a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/00180ac12e6941419c30642a2322014a 2024-12-04T14:56:52,481 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/00180ac12e6941419c30642a2322014a, entries=150, sequenceid=331, filesize=12.0 K 2024-12-04T14:56:52,482 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=33.54 KB/34350 for fc83efc20bfad49ed809f1d485751cce in 1710ms, sequenceid=331, compaction requested=false 2024-12-04T14:56:52,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for fc83efc20bfad49ed809f1d485751cce: 2024-12-04T14:56:52,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:52,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-04T14:56:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-04T14:56:52,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-04T14:56:52,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4760 sec 2024-12-04T14:56:52,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.4810 sec 2024-12-04T14:56:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T14:56:54,186 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4715 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4649 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2140 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6415 rows 2024-12-04T14:56:54,186 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2145 2024-12-04T14:56:54,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6434 rows 2024-12-04T14:56:54,187 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:56:54,187 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:56:54,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:54,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:54,187 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:56:54,188 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T14:56:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:56:54,190 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324214190"}]},"ts":"1733324214190"} 2024-12-04T14:56:54,191 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T14:56:54,192 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T14:56:54,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:56:54,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, UNASSIGN}] 2024-12-04T14:56:54,194 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, UNASSIGN 2024-12-04T14:56:54,194 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=fc83efc20bfad49ed809f1d485751cce, regionState=CLOSING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:54,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, UNASSIGN because future has completed 2024-12-04T14:56:54,196 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:56:54,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:56:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:56:54,349 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:54,349 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:56:54,349 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing fc83efc20bfad49ed809f1d485751cce, disabling compactions & flushes 2024-12-04T14:56:54,349 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:54,349 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:54,349 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. after waiting 0 ms 2024-12-04T14:56:54,350 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:54,350 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing fc83efc20bfad49ed809f1d485751cce 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-04T14:56:54,362 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d9c246cd614d462bae540db00d7ea02c_fc83efc20bfad49ed809f1d485751cce is 50, key is test_row_0/A:col10/1733324211225/Put/seqid=0 2024-12-04T14:56:54,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742028_1204 (size=9914) 2024-12-04T14:56:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:56:54,767 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:54,774 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d9c246cd614d462bae540db00d7ea02c_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d9c246cd614d462bae540db00d7ea02c_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:54,776 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/6e28670f7baa474c87ca00f5560cc1f5, store: [table=TestAcidGuarantees family=A region=fc83efc20bfad49ed809f1d485751cce] 2024-12-04T14:56:54,777 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/6e28670f7baa474c87ca00f5560cc1f5 is 175, key is test_row_0/A:col10/1733324211225/Put/seqid=0 2024-12-04T14:56:54,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742029_1205 (size=22561) 2024-12-04T14:56:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:56:55,184 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/6e28670f7baa474c87ca00f5560cc1f5 2024-12-04T14:56:55,198 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/703c9d9a15284ab7b7d96a9a6dc48779 is 50, key is test_row_0/B:col10/1733324211225/Put/seqid=0 2024-12-04T14:56:55,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742030_1206 (size=9857) 2024-12-04T14:56:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:56:55,603 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/703c9d9a15284ab7b7d96a9a6dc48779 2024-12-04T14:56:55,615 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/5a551b91ffb444f495c504cf30db48ce is 50, key is test_row_0/C:col10/1733324211225/Put/seqid=0 2024-12-04T14:56:55,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742031_1207 (size=9857) 2024-12-04T14:56:56,021 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/5a551b91ffb444f495c504cf30db48ce 2024-12-04T14:56:56,027 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/A/6e28670f7baa474c87ca00f5560cc1f5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6e28670f7baa474c87ca00f5560cc1f5 2024-12-04T14:56:56,032 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6e28670f7baa474c87ca00f5560cc1f5, entries=100, sequenceid=342, filesize=22.0 K 2024-12-04T14:56:56,034 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/B/703c9d9a15284ab7b7d96a9a6dc48779 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/703c9d9a15284ab7b7d96a9a6dc48779 2024-12-04T14:56:56,038 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/703c9d9a15284ab7b7d96a9a6dc48779, entries=100, sequenceid=342, filesize=9.6 K 2024-12-04T14:56:56,039 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/.tmp/C/5a551b91ffb444f495c504cf30db48ce as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/5a551b91ffb444f495c504cf30db48ce 2024-12-04T14:56:56,043 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/5a551b91ffb444f495c504cf30db48ce, entries=100, sequenceid=342, filesize=9.6 K 2024-12-04T14:56:56,044 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fc83efc20bfad49ed809f1d485751cce in 1694ms, sequenceid=342, compaction requested=true 2024-12-04T14:56:56,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/3e440767e17e455b8800edded75b332d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/cfb1786642a7493d8113938434fab250, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/b5a6c84665fa4626bc26df987dda24f6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/dff2a2688e114961941b70be0c4c974c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9cb7c3e156f541748ee77c642e70e5b2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/54f29af579f44c95a7b6a7e6d4a079a7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9140204732dc4693840edd9640eba3e2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/d90bf07e5b1b43fa970a86a9ea31e5f2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e31faf28402b4ab7923117634723d1e9, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/863c30e46d154eaab57b5208e6973f71, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/ae5a7732bf1546d7a324e4a39214dbdd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/71f2fc52a5cd48eeaf654c96c0b5fcf9, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/0db854204aa44e15b01103040fbc5451, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1b3d91190c284f27bbd8af04d26a323c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6be0425581654a67b0fbf79ea8712552, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/7dc2740c206242a0a0858808118e1c3e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1a2f88feea3e4b889318439eca65b64b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e3c4a4be4d0147808e85cfa8e62452af, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/373f0689bcaf40fab59610fe8b40bc3b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/29789b64cf8e4d74b85d74b58864ed01, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/146137fad8414b399e051d531873baa7] to archive 2024-12-04T14:56:56,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:56:56,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/3e440767e17e455b8800edded75b332d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/3e440767e17e455b8800edded75b332d 2024-12-04T14:56:56,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/cfb1786642a7493d8113938434fab250 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/cfb1786642a7493d8113938434fab250 2024-12-04T14:56:56,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/b5a6c84665fa4626bc26df987dda24f6 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/b5a6c84665fa4626bc26df987dda24f6 2024-12-04T14:56:56,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/dff2a2688e114961941b70be0c4c974c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/dff2a2688e114961941b70be0c4c974c 2024-12-04T14:56:56,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9cb7c3e156f541748ee77c642e70e5b2 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9cb7c3e156f541748ee77c642e70e5b2 2024-12-04T14:56:56,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/54f29af579f44c95a7b6a7e6d4a079a7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/54f29af579f44c95a7b6a7e6d4a079a7 2024-12-04T14:56:56,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9140204732dc4693840edd9640eba3e2 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/9140204732dc4693840edd9640eba3e2 2024-12-04T14:56:56,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/d90bf07e5b1b43fa970a86a9ea31e5f2 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/d90bf07e5b1b43fa970a86a9ea31e5f2 2024-12-04T14:56:56,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e31faf28402b4ab7923117634723d1e9 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e31faf28402b4ab7923117634723d1e9 2024-12-04T14:56:56,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/863c30e46d154eaab57b5208e6973f71 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/863c30e46d154eaab57b5208e6973f71 2024-12-04T14:56:56,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/ae5a7732bf1546d7a324e4a39214dbdd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/ae5a7732bf1546d7a324e4a39214dbdd 2024-12-04T14:56:56,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/71f2fc52a5cd48eeaf654c96c0b5fcf9 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/71f2fc52a5cd48eeaf654c96c0b5fcf9 2024-12-04T14:56:56,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/0db854204aa44e15b01103040fbc5451 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/0db854204aa44e15b01103040fbc5451 2024-12-04T14:56:56,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1b3d91190c284f27bbd8af04d26a323c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1b3d91190c284f27bbd8af04d26a323c 2024-12-04T14:56:56,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6be0425581654a67b0fbf79ea8712552 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6be0425581654a67b0fbf79ea8712552 2024-12-04T14:56:56,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/7dc2740c206242a0a0858808118e1c3e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/7dc2740c206242a0a0858808118e1c3e 2024-12-04T14:56:56,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1a2f88feea3e4b889318439eca65b64b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/1a2f88feea3e4b889318439eca65b64b 2024-12-04T14:56:56,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e3c4a4be4d0147808e85cfa8e62452af to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/e3c4a4be4d0147808e85cfa8e62452af 2024-12-04T14:56:56,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/373f0689bcaf40fab59610fe8b40bc3b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/373f0689bcaf40fab59610fe8b40bc3b 2024-12-04T14:56:56,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/29789b64cf8e4d74b85d74b58864ed01 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/29789b64cf8e4d74b85d74b58864ed01 2024-12-04T14:56:56,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/146137fad8414b399e051d531873baa7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/146137fad8414b399e051d531873baa7 2024-12-04T14:56:56,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8fd92e127023497bb65d402a07dd6261, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/b2143e4963104f32be328989dc224659, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8dd74bc7ad784ffc9ae02827108beada, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/4caf7b8565eb4e89904d9e07caab4939, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8674f924d4004c06a107731c91a7eb6e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/893bb734a27742e29be7085863686510, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ec37aef6aeae4165901207c7589b1be5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c2f0e89cb1e04931bfc43f7f0c538829, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/e38306a45eaf4e39a6945d2a5d8a1acc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/473fb89f8fa443cab0a0d69bfa05a0ea, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fd9cc5b1285f40928f97c7e86719a08e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c9943a49c1514a3489fdfc801c18f7f9, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ece534c299264333a2c5d1654c8d71de, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/607933b19e3e43b2bc708269c577c1fa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/73d10c1f154f4bb69e94987005ea8ce3, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/22d87ab1304c4995a736d3414cc8d92f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fdb6ae5283a2441b927ef8ebc3d95460, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/7f159cc9a57b4db68dc0f0e497a87fac, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1fd6d54a27f9495fa8d955ecf430a389, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/eafddb03c7e94acf94d84da577b0452b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1a78cbb315974ddea7aa831287d37c4d] to archive 2024-12-04T14:56:56,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:56:56,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8fd92e127023497bb65d402a07dd6261 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8fd92e127023497bb65d402a07dd6261 2024-12-04T14:56:56,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/b2143e4963104f32be328989dc224659 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/b2143e4963104f32be328989dc224659 2024-12-04T14:56:56,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8dd74bc7ad784ffc9ae02827108beada to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8dd74bc7ad784ffc9ae02827108beada 2024-12-04T14:56:56,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/4caf7b8565eb4e89904d9e07caab4939 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/4caf7b8565eb4e89904d9e07caab4939 2024-12-04T14:56:56,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8674f924d4004c06a107731c91a7eb6e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/8674f924d4004c06a107731c91a7eb6e 2024-12-04T14:56:56,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/893bb734a27742e29be7085863686510 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/893bb734a27742e29be7085863686510 2024-12-04T14:56:56,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ec37aef6aeae4165901207c7589b1be5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ec37aef6aeae4165901207c7589b1be5 2024-12-04T14:56:56,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c2f0e89cb1e04931bfc43f7f0c538829 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c2f0e89cb1e04931bfc43f7f0c538829 2024-12-04T14:56:56,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/e38306a45eaf4e39a6945d2a5d8a1acc to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/e38306a45eaf4e39a6945d2a5d8a1acc 2024-12-04T14:56:56,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/473fb89f8fa443cab0a0d69bfa05a0ea to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/473fb89f8fa443cab0a0d69bfa05a0ea 2024-12-04T14:56:56,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fd9cc5b1285f40928f97c7e86719a08e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fd9cc5b1285f40928f97c7e86719a08e 2024-12-04T14:56:56,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c9943a49c1514a3489fdfc801c18f7f9 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/c9943a49c1514a3489fdfc801c18f7f9 2024-12-04T14:56:56,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ece534c299264333a2c5d1654c8d71de to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/ece534c299264333a2c5d1654c8d71de 2024-12-04T14:56:56,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/607933b19e3e43b2bc708269c577c1fa to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/607933b19e3e43b2bc708269c577c1fa 2024-12-04T14:56:56,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/73d10c1f154f4bb69e94987005ea8ce3 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/73d10c1f154f4bb69e94987005ea8ce3 2024-12-04T14:56:56,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/22d87ab1304c4995a736d3414cc8d92f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/22d87ab1304c4995a736d3414cc8d92f 2024-12-04T14:56:56,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fdb6ae5283a2441b927ef8ebc3d95460 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/fdb6ae5283a2441b927ef8ebc3d95460 2024-12-04T14:56:56,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/7f159cc9a57b4db68dc0f0e497a87fac to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/7f159cc9a57b4db68dc0f0e497a87fac 2024-12-04T14:56:56,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1fd6d54a27f9495fa8d955ecf430a389 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1fd6d54a27f9495fa8d955ecf430a389 2024-12-04T14:56:56,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/eafddb03c7e94acf94d84da577b0452b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/eafddb03c7e94acf94d84da577b0452b 2024-12-04T14:56:56,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1a78cbb315974ddea7aa831287d37c4d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/1a78cbb315974ddea7aa831287d37c4d 2024-12-04T14:56:56,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7bfd0bc5eb5d4de3b73891789105aa67, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7121f53c3fcd47d9a23152ab14e314c7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/92de3235912c408eb9cfec07e4f3f945, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/620b7539bcb74e278a927f365fca7604, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/bd38bf1be157498f89bb11ea0a80f2e7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e5419f9ae55c44e4bc1f5297ba5f95d3, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f87d82de0c294e3cac44bee715558f31, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e358656bee7b4f9e8043f17f77e34a52, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7af0f2a7206a4d22a5321ce306059758, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/035e317f2b2145aab30c72250f9d03ec, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cda103d6ebf547899524ed5690d48422, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/c0f85c48400346ddb2981a53ba65cf28, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/2d608a24c14c4e2cb1ee6d7fd7bb06e7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/198975da3b9546eb9a8edf6fa6282aa8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f0814bf70758435394542364cefa9fef, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8c895dc667b74cc2b41ddc8938e0359e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cb7d82a8c34a465a9065b598d8fa3da6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/01773999983842ada963f673a998f3ef, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/5de10f4cf3c34a3ca31708e3430fab97, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8619f02f73ea4c479828068ab78cff1e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/d30a85db3431402c9caaa899b74b9e9c] to archive 2024-12-04T14:56:56,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:56:56,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7bfd0bc5eb5d4de3b73891789105aa67 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7bfd0bc5eb5d4de3b73891789105aa67 2024-12-04T14:56:56,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7121f53c3fcd47d9a23152ab14e314c7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7121f53c3fcd47d9a23152ab14e314c7 2024-12-04T14:56:56,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/92de3235912c408eb9cfec07e4f3f945 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/92de3235912c408eb9cfec07e4f3f945 2024-12-04T14:56:56,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/620b7539bcb74e278a927f365fca7604 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/620b7539bcb74e278a927f365fca7604 2024-12-04T14:56:56,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/bd38bf1be157498f89bb11ea0a80f2e7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/bd38bf1be157498f89bb11ea0a80f2e7 2024-12-04T14:56:56,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e5419f9ae55c44e4bc1f5297ba5f95d3 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e5419f9ae55c44e4bc1f5297ba5f95d3 2024-12-04T14:56:56,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f87d82de0c294e3cac44bee715558f31 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f87d82de0c294e3cac44bee715558f31 2024-12-04T14:56:56,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e358656bee7b4f9e8043f17f77e34a52 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/e358656bee7b4f9e8043f17f77e34a52 2024-12-04T14:56:56,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7af0f2a7206a4d22a5321ce306059758 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/7af0f2a7206a4d22a5321ce306059758 2024-12-04T14:56:56,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/035e317f2b2145aab30c72250f9d03ec to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/035e317f2b2145aab30c72250f9d03ec 2024-12-04T14:56:56,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cda103d6ebf547899524ed5690d48422 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cda103d6ebf547899524ed5690d48422 2024-12-04T14:56:56,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/c0f85c48400346ddb2981a53ba65cf28 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/c0f85c48400346ddb2981a53ba65cf28 2024-12-04T14:56:56,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/2d608a24c14c4e2cb1ee6d7fd7bb06e7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/2d608a24c14c4e2cb1ee6d7fd7bb06e7 2024-12-04T14:56:56,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/198975da3b9546eb9a8edf6fa6282aa8 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/198975da3b9546eb9a8edf6fa6282aa8 2024-12-04T14:56:56,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f0814bf70758435394542364cefa9fef to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f0814bf70758435394542364cefa9fef 2024-12-04T14:56:56,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8c895dc667b74cc2b41ddc8938e0359e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8c895dc667b74cc2b41ddc8938e0359e 2024-12-04T14:56:56,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cb7d82a8c34a465a9065b598d8fa3da6 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/cb7d82a8c34a465a9065b598d8fa3da6 2024-12-04T14:56:56,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/01773999983842ada963f673a998f3ef to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/01773999983842ada963f673a998f3ef 2024-12-04T14:56:56,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/5de10f4cf3c34a3ca31708e3430fab97 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/5de10f4cf3c34a3ca31708e3430fab97 2024-12-04T14:56:56,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8619f02f73ea4c479828068ab78cff1e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/8619f02f73ea4c479828068ab78cff1e 2024-12-04T14:56:56,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/d30a85db3431402c9caaa899b74b9e9c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/d30a85db3431402c9caaa899b74b9e9c 2024-12-04T14:56:56,120 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/recovered.edits/345.seqid, newMaxSeqId=345, maxSeqId=4 2024-12-04T14:56:56,120 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. 2024-12-04T14:56:56,120 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for fc83efc20bfad49ed809f1d485751cce: Waiting for close lock at 1733324214349Running coprocessor pre-close hooks at 1733324214349Disabling compacts and flushes for region at 1733324214349Disabling writes for close at 1733324214350 (+1 ms)Obtaining lock to block concurrent updates at 1733324214350Preparing flush snapshotting stores in fc83efc20bfad49ed809f1d485751cce at 1733324214350Finished memstore snapshotting TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733324214351 (+1 ms)Flushing stores of TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce. at 1733324214353 (+2 ms)Flushing fc83efc20bfad49ed809f1d485751cce/A: creating writer at 1733324214353Flushing mob file fc83efc20bfad49ed809f1d485751cce/A: appending metadata at 1733324214362 (+9 ms)Flushing mob file fc83efc20bfad49ed809f1d485751cce/A: closing flushed file at 1733324214362Flushing fc83efc20bfad49ed809f1d485751cce/A: appending metadata at 1733324214776 (+414 ms)Flushing fc83efc20bfad49ed809f1d485751cce/A: closing flushed file at 1733324214776Flushing fc83efc20bfad49ed809f1d485751cce/B: creating writer at 1733324215194 (+418 ms)Flushing fc83efc20bfad49ed809f1d485751cce/B: appending metadata at 1733324215197 (+3 ms)Flushing fc83efc20bfad49ed809f1d485751cce/B: closing flushed file at 1733324215197Flushing fc83efc20bfad49ed809f1d485751cce/C: creating writer at 1733324215610 (+413 ms)Flushing fc83efc20bfad49ed809f1d485751cce/C: appending metadata at 1733324215615 (+5 ms)Flushing fc83efc20bfad49ed809f1d485751cce/C: closing flushed file at 1733324215615Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79867998: reopening flushed file at 1733324216026 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39c5f58d: reopening flushed file at 1733324216033 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ee50769: reopening flushed file at 1733324216038 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fc83efc20bfad49ed809f1d485751cce in 1694ms, sequenceid=342, compaction requested=true at 1733324216044 (+6 ms)Writing region close event to WAL at 1733324216117 (+73 ms)Running coprocessor post-close hooks at 1733324216120 (+3 ms)Closed at 1733324216120 2024-12-04T14:56:56,122 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,123 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=fc83efc20bfad49ed809f1d485751cce, regionState=CLOSED 2024-12-04T14:56:56,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:56:56,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-04T14:56:56,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure fc83efc20bfad49ed809f1d485751cce, server=d214edae7bd2,34477,1733324158096 in 1.9290 sec 2024-12-04T14:56:56,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-04T14:56:56,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fc83efc20bfad49ed809f1d485751cce, UNASSIGN in 1.9340 sec 2024-12-04T14:56:56,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-04T14:56:56,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9370 sec 2024-12-04T14:56:56,131 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324216131"}]},"ts":"1733324216131"} 2024-12-04T14:56:56,133 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T14:56:56,133 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T14:56:56,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9450 sec 2024-12-04T14:56:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T14:56:56,336 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:56,338 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T14:56:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:56,343 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T14:56:56,345 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:56,349 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,352 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/recovered.edits] 2024-12-04T14:56:56,356 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/15c30274e8ac4677bdd442f09a2d07e5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/15c30274e8ac4677bdd442f09a2d07e5 2024-12-04T14:56:56,358 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6e28670f7baa474c87ca00f5560cc1f5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/6e28670f7baa474c87ca00f5560cc1f5 2024-12-04T14:56:56,359 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/babc80e0afbe471590b4da8ee7edb59e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/A/babc80e0afbe471590b4da8ee7edb59e 2024-12-04T14:56:56,363 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/703c9d9a15284ab7b7d96a9a6dc48779 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/703c9d9a15284ab7b7d96a9a6dc48779 2024-12-04T14:56:56,365 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/d8b16ea9dac04d1a912999606d032cfb to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/d8b16ea9dac04d1a912999606d032cfb 2024-12-04T14:56:56,367 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/f331ea2634294b28ad73602ad51bd9c6 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/B/f331ea2634294b28ad73602ad51bd9c6 2024-12-04T14:56:56,370 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/00180ac12e6941419c30642a2322014a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/00180ac12e6941419c30642a2322014a 2024-12-04T14:56:56,372 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/5a551b91ffb444f495c504cf30db48ce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/5a551b91ffb444f495c504cf30db48ce 2024-12-04T14:56:56,373 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f93eab560dd74463a764073ed0dbbb13 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/C/f93eab560dd74463a764073ed0dbbb13 2024-12-04T14:56:56,375 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/recovered.edits/345.seqid to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce/recovered.edits/345.seqid 2024-12-04T14:56:56,375 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,375 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T14:56:56,376 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T14:56:56,376 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T14:56:56,380 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040d7e5c4b463a4729a6a35a36a4acbb39_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040d7e5c4b463a4729a6a35a36a4acbb39_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,380 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041e8892263dea4b39a5cd884787dbccb7_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041e8892263dea4b39a5cd884787dbccb7_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,381 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042b9bf28fd764419b91063a4fccc9ac5a_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042b9bf28fd764419b91063a4fccc9ac5a_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,382 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204335ce0baa72f4ec49122c7f866b784ef_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204335ce0baa72f4ec49122c7f866b784ef_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,383 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120435db191c3e1348e8abe652bf4cc50e39_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120435db191c3e1348e8abe652bf4cc50e39_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,384 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043d0a14b633a64eeca44a3e7220e17ab4_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043d0a14b633a64eeca44a3e7220e17ab4_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,385 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045fb1cfa8aa394e1aa8c97d6adf55acf2_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045fb1cfa8aa394e1aa8c97d6adf55acf2_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,386 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204669b0242116544f08ada05c406873ec0_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204669b0242116544f08ada05c406873ec0_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,387 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046ecbdd04135044ae9eaab4469aedf654_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046ecbdd04135044ae9eaab4469aedf654_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,388 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120473c220397b314ba88b4364d088eddf54_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120473c220397b314ba88b4364d088eddf54_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,389 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120475c31684da17434c96b38a128c1bd1e2_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120475c31684da17434c96b38a128c1bd1e2_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,390 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120478a7620443ef4d13a8b0829e741256fd_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120478a7620443ef4d13a8b0829e741256fd_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,391 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047e5765f70c2a40f88fc03bdd1829c65e_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047e5765f70c2a40f88fc03bdd1829c65e_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,392 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a773ecbfe36f4c13a8899a9055ad8e55_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a773ecbfe36f4c13a8899a9055ad8e55_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,393 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b733d20be7744e1695d5b53e8437dd50_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b733d20be7744e1695d5b53e8437dd50_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,394 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d9c246cd614d462bae540db00d7ea02c_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d9c246cd614d462bae540db00d7ea02c_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,395 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204edcb07c1f4b94e57a6da6d3963020858_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204edcb07c1f4b94e57a6da6d3963020858_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,396 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f95ba4d2b2d64e4b9e31af222c00961e_fc83efc20bfad49ed809f1d485751cce to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f95ba4d2b2d64e4b9e31af222c00961e_fc83efc20bfad49ed809f1d485751cce 2024-12-04T14:56:56,396 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T14:56:56,398 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:56,400 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T14:56:56,402 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T14:56:56,403 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:56,403 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T14:56:56,403 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324216403"}]},"ts":"9223372036854775807"} 2024-12-04T14:56:56,405 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T14:56:56,405 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fc83efc20bfad49ed809f1d485751cce, NAME => 'TestAcidGuarantees,,1733324187545.fc83efc20bfad49ed809f1d485751cce.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T14:56:56,405 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T14:56:56,405 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324216405"}]},"ts":"9223372036854775807"} 2024-12-04T14:56:56,407 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T14:56:56,407 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:56,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-12-04T14:56:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T14:56:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T14:56:56,455 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:56,463 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=245 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1963293955_22 at /127.0.0.1:42588 [Waiting for operation #807] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1963293955_22 at /127.0.0.1:42586 [Waiting for operation #830] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2070237321_22 at /127.0.0.1:42462 [Waiting for operation #856] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/cluster_654e7680-3d13-2f9f-e580-aa09589746a6/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/cluster_654e7680-3d13-2f9f-e580-aa09589746a6/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2070237321_22 at /127.0.0.1:40366 [Waiting for operation #1024] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=352 (was 308) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8805 (was 8995) 2024-12-04T14:56:56,471 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=245, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=352, ProcessCount=11, AvailableMemoryMB=8805 2024-12-04T14:56:56,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:56:56,473 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:56:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T14:56:56,475 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:56:56,475 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:56,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-12-04T14:56:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-04T14:56:56,476 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:56:56,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742032_1208 (size=959) 2024-12-04T14:56:56,577 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:56:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-04T14:56:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-04T14:56:56,888 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1 2024-12-04T14:56:56,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742033_1209 (size=53) 2024-12-04T14:56:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-04T14:56:57,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:56:57,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0fa368e0aeee714f048ec4ca8c747b4d, disabling compactions & flushes 2024-12-04T14:56:57,300 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:57,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:57,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. after waiting 0 ms 2024-12-04T14:56:57,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:57,301 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:57,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0fa368e0aeee714f048ec4ca8c747b4d: Waiting for close lock at 1733324217300Disabling compacts and flushes for region at 1733324217300Disabling writes for close at 1733324217301 (+1 ms)Writing region close event to WAL at 1733324217301Closed at 1733324217301 2024-12-04T14:56:57,334 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:56:57,334 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324217334"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324217334"}]},"ts":"1733324217334"} 2024-12-04T14:56:57,338 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T14:56:57,340 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:56:57,341 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324217340"}]},"ts":"1733324217340"} 2024-12-04T14:56:57,344 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T14:56:57,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa368e0aeee714f048ec4ca8c747b4d, ASSIGN}] 2024-12-04T14:56:57,345 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa368e0aeee714f048ec4ca8c747b4d, ASSIGN 2024-12-04T14:56:57,346 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa368e0aeee714f048ec4ca8c747b4d, ASSIGN; state=OFFLINE, location=d214edae7bd2,34477,1733324158096; forceNewPlan=false, retain=false 2024-12-04T14:56:57,498 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=0fa368e0aeee714f048ec4ca8c747b4d, regionState=OPENING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:57,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa368e0aeee714f048ec4ca8c747b4d, ASSIGN because future has completed 2024-12-04T14:56:57,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:56:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-04T14:56:57,664 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:57,664 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:56:57,665 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,665 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:56:57,665 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,665 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,669 INFO [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,672 INFO [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa368e0aeee714f048ec4ca8c747b4d columnFamilyName A 2024-12-04T14:56:57,672 DEBUG [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:57,673 INFO [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] regionserver.HStore(327): Store=0fa368e0aeee714f048ec4ca8c747b4d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:57,674 INFO [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,675 INFO [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa368e0aeee714f048ec4ca8c747b4d columnFamilyName B 2024-12-04T14:56:57,675 DEBUG [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:57,676 INFO [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] regionserver.HStore(327): Store=0fa368e0aeee714f048ec4ca8c747b4d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:57,676 INFO [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,678 INFO [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa368e0aeee714f048ec4ca8c747b4d columnFamilyName C 2024-12-04T14:56:57,678 DEBUG [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:56:57,679 INFO [StoreOpener-0fa368e0aeee714f048ec4ca8c747b4d-1 {}] regionserver.HStore(327): Store=0fa368e0aeee714f048ec4ca8c747b4d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:56:57,679 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,680 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,681 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,683 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,683 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,683 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:56:57,685 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,688 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:56:57,688 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 0fa368e0aeee714f048ec4ca8c747b4d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68029929, jitterRate=0.013724938035011292}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:56:57,688 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:57,689 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 0fa368e0aeee714f048ec4ca8c747b4d: Running coprocessor pre-open hook at 1733324217666Writing region info on filesystem at 1733324217666Initializing all the Stores at 1733324217668 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324217668Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324217668Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324217668Cleaning up temporary data from old regions at 1733324217683 (+15 ms)Running coprocessor post-open hooks at 1733324217688 (+5 ms)Region opened successfully at 1733324217689 (+1 ms) 2024-12-04T14:56:57,690 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., pid=65, masterSystemTime=1733324217657 2024-12-04T14:56:57,691 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:57,691 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:57,692 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=0fa368e0aeee714f048ec4ca8c747b4d, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:57,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:56:57,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-04T14:56:57,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 in 191 msec 2024-12-04T14:56:57,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-04T14:56:57,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa368e0aeee714f048ec4ca8c747b4d, ASSIGN in 351 msec 2024-12-04T14:56:57,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:56:57,697 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324217697"}]},"ts":"1733324217697"} 2024-12-04T14:56:57,699 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T14:56:57,699 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:56:57,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2260 sec 2024-12-04T14:56:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-04T14:56:58,626 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:56:58,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,632 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218d604b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,638 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@624570b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,639 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,642 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:56:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,45797,1733324157426 2024-12-04T14:56:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4844c94d 2024-12-04T14:56:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:56:58,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:56:58,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,646 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f0e67b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,647 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@669a478d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,650 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e3c744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,652 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c99c46a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,654 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,655 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbd6a9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,657 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d64eee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,659 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@724375ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,660 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605154cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,662 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391716d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,664 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21891585, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c46a21f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,667 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a806ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254715a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a8b5a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,671 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34d36860, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e170bb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,674 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513077d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,675 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362a2e63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:56:58,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:56:58,677 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:56:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:56:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:56:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a8a4f8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:56:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:56:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:56:58,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:56:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29444a21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:56:58,679 DEBUG [Thread-1007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,679 DEBUG [Thread-998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,679 DEBUG [Thread-1005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,679 DEBUG [Thread-994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,679 DEBUG [Thread-1011 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,679 DEBUG [Thread-1002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,679 DEBUG [Thread-996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:56:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,680 DEBUG [Thread-1009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,680 DEBUG [Thread-1000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,680 DEBUG [Thread-1013 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:56:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-04T14:56:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:56:58,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:56:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:58,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:56:58,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:58,684 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:56:58,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:58,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:58,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:58,685 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:56:58,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:58,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:56:58,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:58,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:58,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:58,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:56:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:56:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:58,692 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:56:58,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-04T14:56:58,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/1298c8a1afa44de2905c9e336f4ffc57 is 50, key is test_row_0/A:col10/1733324218693/Put/seqid=0 2024-12-04T14:56:58,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324278716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324278720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:58,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324278721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:58,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324278722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:58,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324278722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:58,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742034_1210 (size=12001) 2024-12-04T14:56:58,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/1298c8a1afa44de2905c9e336f4ffc57 2024-12-04T14:56:58,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/fcf7f24835c6450d894ffb2ea3aa0e98 is 50, key is test_row_0/B:col10/1733324218693/Put/seqid=0 2024-12-04T14:56:58,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742035_1211 (size=12001) 2024-12-04T14:56:58,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/fcf7f24835c6450d894ffb2ea3aa0e98 2024-12-04T14:56:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:56:58,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/11cb0c75b4ec4ea0955bae099a205062 is 50, key is test_row_0/C:col10/1733324218693/Put/seqid=0 2024-12-04T14:56:58,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742036_1212 (size=12001) 2024-12-04T14:56:58,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324278825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:58,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:58,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324278835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324278835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324278835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324278836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:58,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:58,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:58,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T14:56:58,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:58,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:56:58,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:58,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:58,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T14:56:58,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:58,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:56:58,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:58,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:58,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:58,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:56:59,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324279035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324279044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324279045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324279045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324279045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T14:56:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:56:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:59,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:59,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:56:59,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/11cb0c75b4ec4ea0955bae099a205062 2024-12-04T14:56:59,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/1298c8a1afa44de2905c9e336f4ffc57 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1298c8a1afa44de2905c9e336f4ffc57 2024-12-04T14:56:59,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1298c8a1afa44de2905c9e336f4ffc57, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T14:56:59,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/fcf7f24835c6450d894ffb2ea3aa0e98 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fcf7f24835c6450d894ffb2ea3aa0e98 2024-12-04T14:56:59,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fcf7f24835c6450d894ffb2ea3aa0e98, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T14:56:59,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/11cb0c75b4ec4ea0955bae099a205062 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/11cb0c75b4ec4ea0955bae099a205062 2024-12-04T14:56:59,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/11cb0c75b4ec4ea0955bae099a205062, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T14:56:59,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 0fa368e0aeee714f048ec4ca8c747b4d in 541ms, sequenceid=14, compaction requested=false 2024-12-04T14:56:59,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:56:59,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T14:56:59,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:56:59,298 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-04T14:56:59,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/af70a43683bb4afdb3255c3dca86634c is 50, key is test_row_0/A:col10/1733324218715/Put/seqid=0 2024-12-04T14:56:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:56:59,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742037_1213 (size=12001) 2024-12-04T14:56:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:56:59,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:56:59,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324279383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324279383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324279383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324279384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324279384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324279495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324279495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324279495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324279495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324279495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324279705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,706 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/af70a43683bb4afdb3255c3dca86634c 2024-12-04T14:56:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324279705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324279705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:56:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324279705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324279705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:56:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:56:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:56:59,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/65b15febf8e34276bd34a27b08bda93a is 50, key is test_row_0/B:col10/1733324218715/Put/seqid=0 2024-12-04T14:56:59,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742038_1214 (size=12001) 2024-12-04T14:56:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:57:00,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324280014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:00,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324280015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:00,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:00,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324280015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324280015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:00,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324280025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:00,132 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/65b15febf8e34276bd34a27b08bda93a 2024-12-04T14:57:00,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/5ddc5b69220340fbb1371420bdc93b5e is 50, key is test_row_0/C:col10/1733324218715/Put/seqid=0 2024-12-04T14:57:00,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742039_1215 (size=12001) 2024-12-04T14:57:00,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324280525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324280525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324280525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:00,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324280525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:00,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324280535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:00,548 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/5ddc5b69220340fbb1371420bdc93b5e 2024-12-04T14:57:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/af70a43683bb4afdb3255c3dca86634c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af70a43683bb4afdb3255c3dca86634c 2024-12-04T14:57:00,556 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af70a43683bb4afdb3255c3dca86634c, entries=150, sequenceid=38, filesize=11.7 K 2024-12-04T14:57:00,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/65b15febf8e34276bd34a27b08bda93a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/65b15febf8e34276bd34a27b08bda93a 2024-12-04T14:57:00,561 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/65b15febf8e34276bd34a27b08bda93a, entries=150, sequenceid=38, filesize=11.7 K 2024-12-04T14:57:00,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/5ddc5b69220340fbb1371420bdc93b5e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ddc5b69220340fbb1371420bdc93b5e 2024-12-04T14:57:00,567 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ddc5b69220340fbb1371420bdc93b5e, entries=150, sequenceid=38, filesize=11.7 K 2024-12-04T14:57:00,568 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0fa368e0aeee714f048ec4ca8c747b4d in 1269ms, sequenceid=38, compaction requested=false 2024-12-04T14:57:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-04T14:57:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-04T14:57:00,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-04T14:57:00,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-12-04T14:57:00,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.8910 sec 2024-12-04T14:57:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T14:57:00,826 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:00,826 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-04T14:57:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:57:00,828 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:00,828 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:00,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:57:00,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T14:57:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:00,980 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-04T14:57:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/f601d3ffe0d548d4b3c778796703cb88 is 50, key is test_row_0/A:col10/1733324219383/Put/seqid=0 2024-12-04T14:57:00,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742040_1216 (size=12001) 2024-12-04T14:57:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:57:01,388 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/f601d3ffe0d548d4b3c778796703cb88 2024-12-04T14:57:01,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/6d418f6c98874ec0b6836ecaac0fd265 is 50, key is test_row_0/B:col10/1733324219383/Put/seqid=0 2024-12-04T14:57:01,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742041_1217 (size=12001) 2024-12-04T14:57:01,423 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/6d418f6c98874ec0b6836ecaac0fd265 2024-12-04T14:57:01,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/e59bba0b45de4ae49804a23428cde443 is 50, key is test_row_0/C:col10/1733324219383/Put/seqid=0 2024-12-04T14:57:01,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742042_1218 (size=12001) 2024-12-04T14:57:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:57:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:01,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:01,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324281554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324281555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324281556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324281557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324281559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,644 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:57:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324281666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324281665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324281666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324281666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324281666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,835 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/e59bba0b45de4ae49804a23428cde443 2024-12-04T14:57:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/f601d3ffe0d548d4b3c778796703cb88 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f601d3ffe0d548d4b3c778796703cb88 2024-12-04T14:57:01,844 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f601d3ffe0d548d4b3c778796703cb88, entries=150, sequenceid=50, filesize=11.7 K 2024-12-04T14:57:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/6d418f6c98874ec0b6836ecaac0fd265 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6d418f6c98874ec0b6836ecaac0fd265 2024-12-04T14:57:01,848 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6d418f6c98874ec0b6836ecaac0fd265, entries=150, sequenceid=50, filesize=11.7 K 2024-12-04T14:57:01,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/e59bba0b45de4ae49804a23428cde443 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/e59bba0b45de4ae49804a23428cde443 2024-12-04T14:57:01,853 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/e59bba0b45de4ae49804a23428cde443, entries=150, sequenceid=50, filesize=11.7 K 2024-12-04T14:57:01,854 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0fa368e0aeee714f048ec4ca8c747b4d in 874ms, sequenceid=50, compaction requested=true 2024-12-04T14:57:01,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:01,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:01,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-04T14:57:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-04T14:57:01,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-04T14:57:01,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0270 sec 2024-12-04T14:57:01,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0310 sec 2024-12-04T14:57:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:01,876 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:01,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:57:01,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/a097c537a244446ea86167ce13ef191a is 50, key is test_row_0/A:col10/1733324221553/Put/seqid=0 2024-12-04T14:57:01,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742043_1219 (size=12001) 2024-12-04T14:57:01,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324281882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324281882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324281883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324281884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324281884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T14:57:01,965 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:01,965 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-04T14:57:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:57:01,967 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:01,968 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:01,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:01,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324281995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324281995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324281995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324281995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324281995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:57:02,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:57:02,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:02,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324282205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324282205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:02,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324282205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324282205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:02,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324282206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:57:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:57:02,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/a097c537a244446ea86167ce13ef191a 2024-12-04T14:57:02,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/0227db87db44456d929bd3fb47229165 is 50, key is test_row_0/B:col10/1733324221553/Put/seqid=0 2024-12-04T14:57:02,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742044_1220 (size=12001) 2024-12-04T14:57:02,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:57:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324282515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:02,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324282515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324282515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324282515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324282515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:02,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:57:02,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:02,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:57:02,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:57:02,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:02,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/0227db87db44456d929bd3fb47229165 2024-12-04T14:57:02,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/a84f8c7e4efd46e785c3bac64a31baf8 is 50, key is test_row_0/C:col10/1733324221553/Put/seqid=0 2024-12-04T14:57:02,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742045_1221 (size=12001) 2024-12-04T14:57:02,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:57:02,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:02,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:02,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324283024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324283024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:03,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324283025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:03,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324283025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:03,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324283025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:03,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:57:03,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:03,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:03,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:03,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:03,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:57:03,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/a84f8c7e4efd46e785c3bac64a31baf8 2024-12-04T14:57:03,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/a097c537a244446ea86167ce13ef191a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/a097c537a244446ea86167ce13ef191a 2024-12-04T14:57:03,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/a097c537a244446ea86167ce13ef191a, entries=150, sequenceid=76, filesize=11.7 K 2024-12-04T14:57:03,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/0227db87db44456d929bd3fb47229165 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0227db87db44456d929bd3fb47229165 2024-12-04T14:57:03,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:57:03,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:03,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:03,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:03,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:03,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:03,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0227db87db44456d929bd3fb47229165, entries=150, sequenceid=76, filesize=11.7 K 2024-12-04T14:57:03,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/a84f8c7e4efd46e785c3bac64a31baf8 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/a84f8c7e4efd46e785c3bac64a31baf8 2024-12-04T14:57:03,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/a84f8c7e4efd46e785c3bac64a31baf8, entries=150, sequenceid=76, filesize=11.7 K 2024-12-04T14:57:03,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 0fa368e0aeee714f048ec4ca8c747b4d in 1326ms, sequenceid=76, compaction requested=true 2024-12-04T14:57:03,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:03,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:03,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:03,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:03,202 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:57:03,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:03,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:03,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:03,202 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:57:03,203 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:57:03,203 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:57:03,204 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/A is initiating minor compaction (all files) 2024-12-04T14:57:03,204 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/B is initiating minor compaction (all files) 2024-12-04T14:57:03,204 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/B in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:03,204 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/A in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:03,204 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1298c8a1afa44de2905c9e336f4ffc57, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af70a43683bb4afdb3255c3dca86634c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f601d3ffe0d548d4b3c778796703cb88, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/a097c537a244446ea86167ce13ef191a] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=46.9 K 2024-12-04T14:57:03,204 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fcf7f24835c6450d894ffb2ea3aa0e98, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/65b15febf8e34276bd34a27b08bda93a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6d418f6c98874ec0b6836ecaac0fd265, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0227db87db44456d929bd3fb47229165] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=46.9 K 2024-12-04T14:57:03,204 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1298c8a1afa44de2905c9e336f4ffc57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733324218690 2024-12-04T14:57:03,204 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting fcf7f24835c6450d894ffb2ea3aa0e98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733324218690 2024-12-04T14:57:03,204 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting af70a43683bb4afdb3255c3dca86634c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733324218715 2024-12-04T14:57:03,204 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 65b15febf8e34276bd34a27b08bda93a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733324218715 2024-12-04T14:57:03,205 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting f601d3ffe0d548d4b3c778796703cb88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733324219358 2024-12-04T14:57:03,205 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d418f6c98874ec0b6836ecaac0fd265, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733324219358 2024-12-04T14:57:03,205 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting a097c537a244446ea86167ce13ef191a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324221553 2024-12-04T14:57:03,205 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 0227db87db44456d929bd3fb47229165, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324221553 2024-12-04T14:57:03,213 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#B#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:03,213 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#A#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:03,213 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/ad73d7bc79f04670b1a065713ee8a643 is 50, key is test_row_0/B:col10/1733324221553/Put/seqid=0 2024-12-04T14:57:03,214 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/1bc76e67f4f94e188f0bad1d6fc3516d is 50, key is test_row_0/A:col10/1733324221553/Put/seqid=0 2024-12-04T14:57:03,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742046_1222 (size=12139) 2024-12-04T14:57:03,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742047_1223 (size=12139) 2024-12-04T14:57:03,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T14:57:03,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:03,345 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:57:03,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/eab325128a264888bcded4e8910d4ebd is 50, key is test_row_0/A:col10/1733324221882/Put/seqid=0 2024-12-04T14:57:03,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742048_1224 (size=12001) 2024-12-04T14:57:03,622 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/ad73d7bc79f04670b1a065713ee8a643 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ad73d7bc79f04670b1a065713ee8a643 2024-12-04T14:57:03,627 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/B of 0fa368e0aeee714f048ec4ca8c747b4d into ad73d7bc79f04670b1a065713ee8a643(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:03,627 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:03,627 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/B, priority=12, startTime=1733324223202; duration=0sec 2024-12-04T14:57:03,627 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:03,627 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:B 2024-12-04T14:57:03,627 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:57:03,629 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:57:03,629 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/C is initiating minor compaction (all files) 2024-12-04T14:57:03,629 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/C in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:03,629 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/11cb0c75b4ec4ea0955bae099a205062, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ddc5b69220340fbb1371420bdc93b5e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/e59bba0b45de4ae49804a23428cde443, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/a84f8c7e4efd46e785c3bac64a31baf8] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=46.9 K 2024-12-04T14:57:03,629 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 11cb0c75b4ec4ea0955bae099a205062, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733324218690 2024-12-04T14:57:03,629 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ddc5b69220340fbb1371420bdc93b5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733324218715 2024-12-04T14:57:03,630 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e59bba0b45de4ae49804a23428cde443, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733324219358 2024-12-04T14:57:03,630 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting a84f8c7e4efd46e785c3bac64a31baf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324221553 2024-12-04T14:57:03,651 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/1bc76e67f4f94e188f0bad1d6fc3516d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1bc76e67f4f94e188f0bad1d6fc3516d 2024-12-04T14:57:03,653 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#C#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:03,653 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/f6f7dc317f144475908ff1845b815727 is 50, key is test_row_0/C:col10/1733324221553/Put/seqid=0 2024-12-04T14:57:03,657 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/A of 0fa368e0aeee714f048ec4ca8c747b4d into 1bc76e67f4f94e188f0bad1d6fc3516d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:03,657 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:03,657 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/A, priority=12, startTime=1733324223202; duration=0sec 2024-12-04T14:57:03,657 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:03,657 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:A 2024-12-04T14:57:03,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742049_1225 (size=12139) 2024-12-04T14:57:03,770 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/eab325128a264888bcded4e8910d4ebd 2024-12-04T14:57:03,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/c51517a4a3b7485aad6207483db597ff is 50, key is test_row_0/B:col10/1733324221882/Put/seqid=0 2024-12-04T14:57:03,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742050_1226 (size=12001) 2024-12-04T14:57:03,783 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/c51517a4a3b7485aad6207483db597ff 2024-12-04T14:57:03,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/4e2f45b81916481c914993385a0dd04a is 50, key is test_row_0/C:col10/1733324221882/Put/seqid=0 2024-12-04T14:57:03,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742051_1227 (size=12001) 2024-12-04T14:57:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:04,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324284059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324284058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324284060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,066 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/f6f7dc317f144475908ff1845b815727 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f6f7dc317f144475908ff1845b815727 2024-12-04T14:57:04,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324284063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324284064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,072 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/C of 0fa368e0aeee714f048ec4ca8c747b4d into f6f7dc317f144475908ff1845b815727(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:04,072 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:04,072 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/C, priority=12, startTime=1733324223202; duration=0sec 2024-12-04T14:57:04,072 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:04,072 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:C 2024-12-04T14:57:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:57:04,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324284165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324284174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324284174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324284175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324284175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,195 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/4e2f45b81916481c914993385a0dd04a 2024-12-04T14:57:04,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/eab325128a264888bcded4e8910d4ebd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/eab325128a264888bcded4e8910d4ebd 2024-12-04T14:57:04,203 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/eab325128a264888bcded4e8910d4ebd, entries=150, sequenceid=87, filesize=11.7 K 2024-12-04T14:57:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/c51517a4a3b7485aad6207483db597ff as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/c51517a4a3b7485aad6207483db597ff 2024-12-04T14:57:04,208 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/c51517a4a3b7485aad6207483db597ff, entries=150, sequenceid=87, filesize=11.7 K 2024-12-04T14:57:04,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/4e2f45b81916481c914993385a0dd04a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4e2f45b81916481c914993385a0dd04a 2024-12-04T14:57:04,212 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4e2f45b81916481c914993385a0dd04a, entries=150, sequenceid=87, filesize=11.7 K 2024-12-04T14:57:04,213 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 0fa368e0aeee714f048ec4ca8c747b4d in 868ms, sequenceid=87, compaction requested=false 2024-12-04T14:57:04,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:04,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:04,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-04T14:57:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-04T14:57:04,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-04T14:57:04,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2470 sec 2024-12-04T14:57:04,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2510 sec 2024-12-04T14:57:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:04,378 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:04,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-04T14:57:04,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/3e0f51359a7740f8b24dd7481e6c71fe is 50, key is test_row_0/A:col10/1733324224058/Put/seqid=0 2024-12-04T14:57:04,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742052_1228 (size=12001) 2024-12-04T14:57:04,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/3e0f51359a7740f8b24dd7481e6c71fe 2024-12-04T14:57:04,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324284388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324284388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324284389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324284390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324284390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/6747c025fac5428cb7563c26b66d9c5c is 50, key is test_row_0/B:col10/1733324224058/Put/seqid=0 2024-12-04T14:57:04,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742053_1229 (size=12001) 2024-12-04T14:57:04,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324284495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324284495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324284505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324284505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324284505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324284705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324284705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324284715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324284715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324284715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:04,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/6747c025fac5428cb7563c26b66d9c5c 2024-12-04T14:57:04,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/3475034df6bb4064b368f30ea4c9fa17 is 50, key is test_row_0/C:col10/1733324224058/Put/seqid=0 2024-12-04T14:57:04,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742054_1230 (size=12001) 2024-12-04T14:57:05,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324285014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324285015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324285025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324285025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324285025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/3475034df6bb4064b368f30ea4c9fa17 2024-12-04T14:57:05,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/3e0f51359a7740f8b24dd7481e6c71fe as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3e0f51359a7740f8b24dd7481e6c71fe 2024-12-04T14:57:05,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3e0f51359a7740f8b24dd7481e6c71fe, entries=150, sequenceid=117, filesize=11.7 K 2024-12-04T14:57:05,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/6747c025fac5428cb7563c26b66d9c5c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6747c025fac5428cb7563c26b66d9c5c 2024-12-04T14:57:05,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6747c025fac5428cb7563c26b66d9c5c, entries=150, sequenceid=117, filesize=11.7 K 2024-12-04T14:57:05,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/3475034df6bb4064b368f30ea4c9fa17 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/3475034df6bb4064b368f30ea4c9fa17 2024-12-04T14:57:05,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/3475034df6bb4064b368f30ea4c9fa17, entries=150, sequenceid=117, filesize=11.7 K 2024-12-04T14:57:05,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 0fa368e0aeee714f048ec4ca8c747b4d in 872ms, sequenceid=117, compaction requested=true 2024-12-04T14:57:05,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:05,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:05,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:05,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:05,251 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:05,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:05,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:05,251 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:05,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:05,252 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:05,252 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:05,252 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/A is initiating minor compaction (all files) 2024-12-04T14:57:05,252 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/B is initiating minor compaction (all files) 2024-12-04T14:57:05,252 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/B in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:05,252 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/A in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:05,252 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1bc76e67f4f94e188f0bad1d6fc3516d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/eab325128a264888bcded4e8910d4ebd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3e0f51359a7740f8b24dd7481e6c71fe] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=35.3 K 2024-12-04T14:57:05,252 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ad73d7bc79f04670b1a065713ee8a643, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/c51517a4a3b7485aad6207483db597ff, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6747c025fac5428cb7563c26b66d9c5c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=35.3 K 2024-12-04T14:57:05,252 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bc76e67f4f94e188f0bad1d6fc3516d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324221553 2024-12-04T14:57:05,253 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting ad73d7bc79f04670b1a065713ee8a643, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324221553 2024-12-04T14:57:05,253 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting eab325128a264888bcded4e8910d4ebd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733324221881 2024-12-04T14:57:05,253 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c51517a4a3b7485aad6207483db597ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733324221881 2024-12-04T14:57:05,253 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e0f51359a7740f8b24dd7481e6c71fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324224058 2024-12-04T14:57:05,253 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 6747c025fac5428cb7563c26b66d9c5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324224058 2024-12-04T14:57:05,260 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#A#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:05,261 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/387aab508e8c4135baab0d417c4dba61 is 50, key is test_row_0/A:col10/1733324224058/Put/seqid=0 2024-12-04T14:57:05,267 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#B#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:05,267 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/0e4196e6b8f047f59fc034cb94a37cfd is 50, key is test_row_0/B:col10/1733324224058/Put/seqid=0 2024-12-04T14:57:05,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742056_1232 (size=12241) 2024-12-04T14:57:05,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742055_1231 (size=12241) 2024-12-04T14:57:05,280 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/387aab508e8c4135baab0d417c4dba61 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/387aab508e8c4135baab0d417c4dba61 2024-12-04T14:57:05,284 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/A of 0fa368e0aeee714f048ec4ca8c747b4d into 387aab508e8c4135baab0d417c4dba61(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:05,284 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:05,284 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/A, priority=13, startTime=1733324225250; duration=0sec 2024-12-04T14:57:05,284 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:05,284 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:A 2024-12-04T14:57:05,284 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:05,285 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:05,285 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/C is initiating minor compaction (all files) 2024-12-04T14:57:05,285 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/C in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:05,285 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f6f7dc317f144475908ff1845b815727, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4e2f45b81916481c914993385a0dd04a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/3475034df6bb4064b368f30ea4c9fa17] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=35.3 K 2024-12-04T14:57:05,286 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6f7dc317f144475908ff1845b815727, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324221553 2024-12-04T14:57:05,286 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e2f45b81916481c914993385a0dd04a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733324221881 2024-12-04T14:57:05,286 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3475034df6bb4064b368f30ea4c9fa17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324224058 2024-12-04T14:57:05,314 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#C#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:05,314 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/34841bd6a4544788a9c4e8d6700abc84 is 50, key is test_row_0/C:col10/1733324224058/Put/seqid=0 2024-12-04T14:57:05,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742057_1233 (size=12241) 2024-12-04T14:57:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:05,527 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:05,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:57:05,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/25ea3705e9c74f6da4097ae5841c6b88 is 50, key is test_row_0/A:col10/1733324225526/Put/seqid=0 2024-12-04T14:57:05,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742058_1234 (size=14441) 2024-12-04T14:57:05,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324285595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324285596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324285597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324285599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324285601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,680 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/0e4196e6b8f047f59fc034cb94a37cfd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0e4196e6b8f047f59fc034cb94a37cfd 2024-12-04T14:57:05,684 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/B of 0fa368e0aeee714f048ec4ca8c747b4d into 0e4196e6b8f047f59fc034cb94a37cfd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:05,685 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:05,685 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/B, priority=13, startTime=1733324225251; duration=0sec 2024-12-04T14:57:05,685 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:05,685 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:B 2024-12-04T14:57:05,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324285705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324285705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324285705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324285705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324285714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,727 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/34841bd6a4544788a9c4e8d6700abc84 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/34841bd6a4544788a9c4e8d6700abc84 2024-12-04T14:57:05,732 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/C of 0fa368e0aeee714f048ec4ca8c747b4d into 34841bd6a4544788a9c4e8d6700abc84(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:05,732 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:05,732 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/C, priority=13, startTime=1733324225251; duration=0sec 2024-12-04T14:57:05,732 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:05,732 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:C 2024-12-04T14:57:05,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324285915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324285915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324285915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324285915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324285925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:05,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/25ea3705e9c74f6da4097ae5841c6b88 2024-12-04T14:57:05,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/2e7efddb0d8d49bdbf42eecbd00a6a28 is 50, key is test_row_0/B:col10/1733324225526/Put/seqid=0 2024-12-04T14:57:05,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742059_1235 (size=12101) 2024-12-04T14:57:05,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/2e7efddb0d8d49bdbf42eecbd00a6a28 2024-12-04T14:57:05,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/6a58041c4cab40da9dd04e6e5c39f4de is 50, key is test_row_0/C:col10/1733324225526/Put/seqid=0 2024-12-04T14:57:05,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742060_1236 (size=12101) 2024-12-04T14:57:05,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/6a58041c4cab40da9dd04e6e5c39f4de 2024-12-04T14:57:05,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/25ea3705e9c74f6da4097ae5841c6b88 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/25ea3705e9c74f6da4097ae5841c6b88 2024-12-04T14:57:05,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/25ea3705e9c74f6da4097ae5841c6b88, entries=200, sequenceid=130, filesize=14.1 K 2024-12-04T14:57:05,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/2e7efddb0d8d49bdbf42eecbd00a6a28 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2e7efddb0d8d49bdbf42eecbd00a6a28 2024-12-04T14:57:05,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2e7efddb0d8d49bdbf42eecbd00a6a28, entries=150, sequenceid=130, filesize=11.8 K 2024-12-04T14:57:05,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/6a58041c4cab40da9dd04e6e5c39f4de as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/6a58041c4cab40da9dd04e6e5c39f4de 2024-12-04T14:57:05,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/6a58041c4cab40da9dd04e6e5c39f4de, entries=150, sequenceid=130, filesize=11.8 K 2024-12-04T14:57:06,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0fa368e0aeee714f048ec4ca8c747b4d in 473ms, sequenceid=130, compaction requested=false 2024-12-04T14:57:06,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T14:57:06,135 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:06,135 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-04T14:57:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:57:06,139 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:06,140 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:06,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:06,227 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:06,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-04T14:57:06,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/154a11dd3e88456587558ebccae6ff1d is 50, key is test_row_0/A:col10/1733324226227/Put/seqid=0 2024-12-04T14:57:06,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324286233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324286234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324286235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324286237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324286239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:57:06,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742061_1237 (size=16931) 2024-12-04T14:57:06,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/154a11dd3e88456587558ebccae6ff1d 2024-12-04T14:57:06,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/fe215c2197d64b18acdf6e7a8dbc4b1d is 50, key is test_row_0/B:col10/1733324226227/Put/seqid=0 2024-12-04T14:57:06,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742062_1238 (size=12151) 2024-12-04T14:57:06,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T14:57:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:06,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:06,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324286345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324286345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324286345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324286345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T14:57:06,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:06,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:06,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:06,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:57:06,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324286555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324286555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324286555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324286555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T14:57:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:06,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/fe215c2197d64b18acdf6e7a8dbc4b1d 2024-12-04T14:57:06,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/408dc63aa6f84c138cbae96b083dbd2f is 50, key is test_row_0/C:col10/1733324226227/Put/seqid=0 2024-12-04T14:57:06,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742063_1239 (size=12151) 2024-12-04T14:57:06,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324286745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T14:57:06,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:06,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:06,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:06,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:57:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324286864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324286864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324286865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324286865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:06,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T14:57:06,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:06,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:06,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:06,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:07,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T14:57:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:07,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:07,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/408dc63aa6f84c138cbae96b083dbd2f 2024-12-04T14:57:07,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/154a11dd3e88456587558ebccae6ff1d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/154a11dd3e88456587558ebccae6ff1d 2024-12-04T14:57:07,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/154a11dd3e88456587558ebccae6ff1d, entries=250, sequenceid=160, filesize=16.5 K 2024-12-04T14:57:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/fe215c2197d64b18acdf6e7a8dbc4b1d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fe215c2197d64b18acdf6e7a8dbc4b1d 2024-12-04T14:57:07,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fe215c2197d64b18acdf6e7a8dbc4b1d, entries=150, sequenceid=160, filesize=11.9 K 2024-12-04T14:57:07,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/408dc63aa6f84c138cbae96b083dbd2f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/408dc63aa6f84c138cbae96b083dbd2f 2024-12-04T14:57:07,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/408dc63aa6f84c138cbae96b083dbd2f, entries=150, sequenceid=160, filesize=11.9 K 2024-12-04T14:57:07,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 0fa368e0aeee714f048ec4ca8c747b4d in 862ms, sequenceid=160, compaction requested=true 2024-12-04T14:57:07,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:07,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:07,090 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:07,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:07,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:07,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:07,090 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:07,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:07,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:07,091 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:07,091 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/B is initiating minor compaction (all files) 2024-12-04T14:57:07,091 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43613 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:07,091 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/A is initiating minor compaction (all files) 2024-12-04T14:57:07,091 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/B in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:07,091 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/A in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:07,091 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0e4196e6b8f047f59fc034cb94a37cfd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2e7efddb0d8d49bdbf42eecbd00a6a28, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fe215c2197d64b18acdf6e7a8dbc4b1d] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=35.6 K 2024-12-04T14:57:07,091 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/387aab508e8c4135baab0d417c4dba61, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/25ea3705e9c74f6da4097ae5841c6b88, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/154a11dd3e88456587558ebccae6ff1d] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=42.6 K 2024-12-04T14:57:07,091 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e4196e6b8f047f59fc034cb94a37cfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324224058 2024-12-04T14:57:07,091 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 387aab508e8c4135baab0d417c4dba61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324224058 2024-12-04T14:57:07,092 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e7efddb0d8d49bdbf42eecbd00a6a28, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733324224387 2024-12-04T14:57:07,092 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25ea3705e9c74f6da4097ae5841c6b88, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733324224387 2024-12-04T14:57:07,092 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting fe215c2197d64b18acdf6e7a8dbc4b1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324226226 2024-12-04T14:57:07,092 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 154a11dd3e88456587558ebccae6ff1d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324225599 2024-12-04T14:57:07,098 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#A#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:07,099 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/d2c6709dcc104a5a91405401802121ba is 50, key is test_row_0/A:col10/1733324226227/Put/seqid=0 2024-12-04T14:57:07,104 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#B#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:07,104 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/dc0a2bef39ca4487864b5377974a71d4 is 50, key is test_row_0/B:col10/1733324226227/Put/seqid=0 2024-12-04T14:57:07,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742064_1240 (size=12493) 2024-12-04T14:57:07,113 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/d2c6709dcc104a5a91405401802121ba as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/d2c6709dcc104a5a91405401802121ba 2024-12-04T14:57:07,119 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/A of 0fa368e0aeee714f048ec4ca8c747b4d into d2c6709dcc104a5a91405401802121ba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:07,119 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:07,119 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/A, priority=13, startTime=1733324227089; duration=0sec 2024-12-04T14:57:07,119 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:07,119 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:A 2024-12-04T14:57:07,119 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:07,120 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:07,121 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/C is initiating minor compaction (all files) 2024-12-04T14:57:07,121 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/C in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:07,121 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/34841bd6a4544788a9c4e8d6700abc84, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/6a58041c4cab40da9dd04e6e5c39f4de, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/408dc63aa6f84c138cbae96b083dbd2f] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=35.6 K 2024-12-04T14:57:07,121 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34841bd6a4544788a9c4e8d6700abc84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324224058 2024-12-04T14:57:07,121 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a58041c4cab40da9dd04e6e5c39f4de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733324224387 2024-12-04T14:57:07,122 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408dc63aa6f84c138cbae96b083dbd2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324226226 2024-12-04T14:57:07,128 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#C#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:07,129 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/85cabd4ee2884d9f982c1525d1874d33 is 50, key is test_row_0/C:col10/1733324226227/Put/seqid=0 2024-12-04T14:57:07,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742065_1241 (size=12493) 2024-12-04T14:57:07,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742066_1242 (size=12493) 2024-12-04T14:57:07,158 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/85cabd4ee2884d9f982c1525d1874d33 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/85cabd4ee2884d9f982c1525d1874d33 2024-12-04T14:57:07,162 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/C of 0fa368e0aeee714f048ec4ca8c747b4d into 85cabd4ee2884d9f982c1525d1874d33(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:07,162 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:07,162 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/C, priority=13, startTime=1733324227090; duration=0sec 2024-12-04T14:57:07,163 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:07,163 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:C 2024-12-04T14:57:07,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T14:57:07,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:07,210 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-04T14:57:07,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/89b1d737a1b84a2ca50cd57ac381c8f8 is 50, key is test_row_0/A:col10/1733324226235/Put/seqid=0 2024-12-04T14:57:07,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742067_1243 (size=12151) 2024-12-04T14:57:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:57:07,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:07,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324287398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324287399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324287400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324287401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324287505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324287505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324287505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324287514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,535 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/dc0a2bef39ca4487864b5377974a71d4 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dc0a2bef39ca4487864b5377974a71d4 2024-12-04T14:57:07,541 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/B of 0fa368e0aeee714f048ec4ca8c747b4d into dc0a2bef39ca4487864b5377974a71d4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:07,541 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:07,541 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/B, priority=13, startTime=1733324227090; duration=0sec 2024-12-04T14:57:07,541 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:07,541 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:B 2024-12-04T14:57:07,621 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/89b1d737a1b84a2ca50cd57ac381c8f8 2024-12-04T14:57:07,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/7c02c161e3e94a7288490bd83dc23662 is 50, key is test_row_0/B:col10/1733324226235/Put/seqid=0 2024-12-04T14:57:07,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742068_1244 (size=12151) 2024-12-04T14:57:07,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324287715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324287715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324287715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324287725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:07,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324287755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:07,756 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0fa368e0aeee714f048ec4ca8c747b4d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324288025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324288025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324288025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324288034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,045 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/7c02c161e3e94a7288490bd83dc23662 2024-12-04T14:57:08,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/d97486b39a3045b1b0f24c548b89d6f7 is 50, key is test_row_0/C:col10/1733324226235/Put/seqid=0 2024-12-04T14:57:08,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742069_1245 (size=12151) 2024-12-04T14:57:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:57:08,455 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/d97486b39a3045b1b0f24c548b89d6f7 2024-12-04T14:57:08,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/89b1d737a1b84a2ca50cd57ac381c8f8 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/89b1d737a1b84a2ca50cd57ac381c8f8 2024-12-04T14:57:08,463 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/89b1d737a1b84a2ca50cd57ac381c8f8, entries=150, sequenceid=170, filesize=11.9 K 2024-12-04T14:57:08,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/7c02c161e3e94a7288490bd83dc23662 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/7c02c161e3e94a7288490bd83dc23662 2024-12-04T14:57:08,469 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/7c02c161e3e94a7288490bd83dc23662, entries=150, sequenceid=170, filesize=11.9 K 2024-12-04T14:57:08,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/d97486b39a3045b1b0f24c548b89d6f7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/d97486b39a3045b1b0f24c548b89d6f7 2024-12-04T14:57:08,474 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/d97486b39a3045b1b0f24c548b89d6f7, entries=150, sequenceid=170, filesize=11.9 K 2024-12-04T14:57:08,475 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 0fa368e0aeee714f048ec4ca8c747b4d in 1264ms, sequenceid=170, compaction requested=false 2024-12-04T14:57:08,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:08,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:08,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-04T14:57:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-04T14:57:08,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-04T14:57:08,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3360 sec 2024-12-04T14:57:08,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.3430 sec 2024-12-04T14:57:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:08,561 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:08,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-12-04T14:57:08,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324288561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324288561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324288562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/6180396ebbfe4d63badad91def6666cd is 50, key is test_row_0/A:col10/1733324228536/Put/seqid=0 2024-12-04T14:57:08,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324288564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742070_1246 (size=12151) 2024-12-04T14:57:08,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324288675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324288675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324288675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324288675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324288885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324288885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324288885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324288885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:08,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/6180396ebbfe4d63badad91def6666cd 2024-12-04T14:57:08,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/fc605d2aeac84cd2a846d94a17f471e6 is 50, key is test_row_0/B:col10/1733324228536/Put/seqid=0 2024-12-04T14:57:09,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742071_1247 (size=12151) 2024-12-04T14:57:09,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/fc605d2aeac84cd2a846d94a17f471e6 2024-12-04T14:57:09,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/08316001014e47d98ea9d3e4cf290eab is 50, key is test_row_0/C:col10/1733324228536/Put/seqid=0 2024-12-04T14:57:09,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742072_1248 (size=12151) 2024-12-04T14:57:09,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/08316001014e47d98ea9d3e4cf290eab 2024-12-04T14:57:09,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/6180396ebbfe4d63badad91def6666cd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/6180396ebbfe4d63badad91def6666cd 2024-12-04T14:57:09,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/6180396ebbfe4d63badad91def6666cd, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T14:57:09,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/fc605d2aeac84cd2a846d94a17f471e6 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fc605d2aeac84cd2a846d94a17f471e6 2024-12-04T14:57:09,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fc605d2aeac84cd2a846d94a17f471e6, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T14:57:09,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/08316001014e47d98ea9d3e4cf290eab as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/08316001014e47d98ea9d3e4cf290eab 2024-12-04T14:57:09,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/08316001014e47d98ea9d3e4cf290eab, entries=150, sequenceid=203, filesize=11.9 K 2024-12-04T14:57:09,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 0fa368e0aeee714f048ec4ca8c747b4d in 497ms, sequenceid=203, compaction requested=true 2024-12-04T14:57:09,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:09,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:09,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:09,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:09,059 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:09,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:09,059 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:09,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:09,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:09,061 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:09,061 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:09,061 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/B is initiating minor compaction (all files) 2024-12-04T14:57:09,061 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/A is initiating minor compaction (all files) 2024-12-04T14:57:09,061 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/A in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:09,061 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/d2c6709dcc104a5a91405401802121ba, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/89b1d737a1b84a2ca50cd57ac381c8f8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/6180396ebbfe4d63badad91def6666cd] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=35.9 K 2024-12-04T14:57:09,061 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/B in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:09,061 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dc0a2bef39ca4487864b5377974a71d4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/7c02c161e3e94a7288490bd83dc23662, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fc605d2aeac84cd2a846d94a17f471e6] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=35.9 K 2024-12-04T14:57:09,061 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2c6709dcc104a5a91405401802121ba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324226226 2024-12-04T14:57:09,061 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting dc0a2bef39ca4487864b5377974a71d4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324226226 2024-12-04T14:57:09,062 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89b1d737a1b84a2ca50cd57ac381c8f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733324226230 2024-12-04T14:57:09,062 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c02c161e3e94a7288490bd83dc23662, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733324226230 2024-12-04T14:57:09,063 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting fc605d2aeac84cd2a846d94a17f471e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324228536 2024-12-04T14:57:09,063 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6180396ebbfe4d63badad91def6666cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324228536 2024-12-04T14:57:09,072 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:09,073 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/2817593ff8a14023ae0a3f424dedf93e is 50, key is test_row_0/B:col10/1733324228536/Put/seqid=0 2024-12-04T14:57:09,078 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#A#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:09,078 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/f80c3c77ad1a4974a5431458d47de66d is 50, key is test_row_0/A:col10/1733324228536/Put/seqid=0 2024-12-04T14:57:09,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742073_1249 (size=12595) 2024-12-04T14:57:09,094 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/2817593ff8a14023ae0a3f424dedf93e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2817593ff8a14023ae0a3f424dedf93e 2024-12-04T14:57:09,100 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/B of 0fa368e0aeee714f048ec4ca8c747b4d into 2817593ff8a14023ae0a3f424dedf93e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:09,100 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:09,100 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/B, priority=13, startTime=1733324229059; duration=0sec 2024-12-04T14:57:09,100 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:09,100 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:B 2024-12-04T14:57:09,100 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:09,101 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:09,102 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/C is initiating minor compaction (all files) 2024-12-04T14:57:09,102 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/C in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:09,102 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/85cabd4ee2884d9f982c1525d1874d33, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/d97486b39a3045b1b0f24c548b89d6f7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/08316001014e47d98ea9d3e4cf290eab] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=35.9 K 2024-12-04T14:57:09,102 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 85cabd4ee2884d9f982c1525d1874d33, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733324226226 2024-12-04T14:57:09,102 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting d97486b39a3045b1b0f24c548b89d6f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733324226230 2024-12-04T14:57:09,103 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 08316001014e47d98ea9d3e4cf290eab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324228536 2024-12-04T14:57:09,113 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#C#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:09,113 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/2edc3fcbe9644af4bde97c6dea715629 is 50, key is test_row_0/C:col10/1733324228536/Put/seqid=0 2024-12-04T14:57:09,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742074_1250 (size=12595) 2024-12-04T14:57:09,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742075_1251 (size=12595) 2024-12-04T14:57:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:09,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:09,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-04T14:57:09,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/5a1fa1bdd8d94237a5073f69fe5cea84 is 50, key is test_row_0/A:col10/1733324229221/Put/seqid=0 2024-12-04T14:57:09,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742076_1252 (size=16931) 2024-12-04T14:57:09,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324289267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324289268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324289269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324289270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324289374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324289374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324289375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324289375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,520 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/f80c3c77ad1a4974a5431458d47de66d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f80c3c77ad1a4974a5431458d47de66d 2024-12-04T14:57:09,524 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/A of 0fa368e0aeee714f048ec4ca8c747b4d into f80c3c77ad1a4974a5431458d47de66d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:09,524 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:09,524 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/A, priority=13, startTime=1733324229059; duration=0sec 2024-12-04T14:57:09,524 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:09,524 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:A 2024-12-04T14:57:09,527 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/2edc3fcbe9644af4bde97c6dea715629 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/2edc3fcbe9644af4bde97c6dea715629 2024-12-04T14:57:09,531 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/C of 0fa368e0aeee714f048ec4ca8c747b4d into 2edc3fcbe9644af4bde97c6dea715629(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:09,531 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:09,531 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/C, priority=13, startTime=1733324229059; duration=0sec 2024-12-04T14:57:09,531 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:09,531 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:C 2024-12-04T14:57:09,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324289584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324289584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324289585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324289585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/5a1fa1bdd8d94237a5073f69fe5cea84 2024-12-04T14:57:09,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/98e6f755f0a34d4d86857b14b7d96115 is 50, key is test_row_0/B:col10/1733324229221/Put/seqid=0 2024-12-04T14:57:09,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742077_1253 (size=12151) 2024-12-04T14:57:09,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324289775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,777 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0fa368e0aeee714f048ec4ca8c747b4d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324289895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324289895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324289895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324289896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:10,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/98e6f755f0a34d4d86857b14b7d96115 2024-12-04T14:57:10,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/c924ae000ddb4182ad218db19915b276 is 50, key is test_row_0/C:col10/1733324229221/Put/seqid=0 2024-12-04T14:57:10,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742078_1254 (size=12151) 2024-12-04T14:57:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T14:57:10,305 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:10,305 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-04T14:57:10,307 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:57:10,307 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:10,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:10,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324290404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:10,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324290405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:10,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324290405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:10,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324290405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:57:10,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/c924ae000ddb4182ad218db19915b276 2024-12-04T14:57:10,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/5a1fa1bdd8d94237a5073f69fe5cea84 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/5a1fa1bdd8d94237a5073f69fe5cea84 2024-12-04T14:57:10,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T14:57:10,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:10,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:10,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:10,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/5a1fa1bdd8d94237a5073f69fe5cea84, entries=250, sequenceid=216, filesize=16.5 K 2024-12-04T14:57:10,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:10,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:10,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/98e6f755f0a34d4d86857b14b7d96115 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/98e6f755f0a34d4d86857b14b7d96115 2024-12-04T14:57:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:10,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/98e6f755f0a34d4d86857b14b7d96115, entries=150, sequenceid=216, filesize=11.9 K 2024-12-04T14:57:10,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/c924ae000ddb4182ad218db19915b276 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/c924ae000ddb4182ad218db19915b276 2024-12-04T14:57:10,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/c924ae000ddb4182ad218db19915b276, entries=150, sequenceid=216, filesize=11.9 K 2024-12-04T14:57:10,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0fa368e0aeee714f048ec4ca8c747b4d in 1248ms, sequenceid=216, compaction requested=false 2024-12-04T14:57:10,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:10,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T14:57:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:10,614 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-04T14:57:10,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/44ae287002414758958d63be8788bfb3 is 50, key is test_row_0/A:col10/1733324229269/Put/seqid=0 2024-12-04T14:57:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:57:10,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742079_1255 (size=12151) 2024-12-04T14:57:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:57:11,075 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/44ae287002414758958d63be8788bfb3 2024-12-04T14:57:11,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/a3a6b16cdf9f4f9391fafc5c71efc03b is 50, key is test_row_0/B:col10/1733324229269/Put/seqid=0 2024-12-04T14:57:11,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742080_1256 (size=12151) 2024-12-04T14:57:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:11,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324291426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324291429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324291429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324291430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:57:11,487 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/a3a6b16cdf9f4f9391fafc5c71efc03b 2024-12-04T14:57:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/4bd98c3d7a12402e9c2f289b412c1966 is 50, key is test_row_0/C:col10/1733324229269/Put/seqid=0 2024-12-04T14:57:11,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742081_1257 (size=12151) 2024-12-04T14:57:11,501 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/4bd98c3d7a12402e9c2f289b412c1966 2024-12-04T14:57:11,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/44ae287002414758958d63be8788bfb3 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/44ae287002414758958d63be8788bfb3 2024-12-04T14:57:11,509 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/44ae287002414758958d63be8788bfb3, entries=150, sequenceid=242, filesize=11.9 K 2024-12-04T14:57:11,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/a3a6b16cdf9f4f9391fafc5c71efc03b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a3a6b16cdf9f4f9391fafc5c71efc03b 2024-12-04T14:57:11,526 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a3a6b16cdf9f4f9391fafc5c71efc03b, entries=150, sequenceid=242, filesize=11.9 K 2024-12-04T14:57:11,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/4bd98c3d7a12402e9c2f289b412c1966 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4bd98c3d7a12402e9c2f289b412c1966 2024-12-04T14:57:11,534 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4bd98c3d7a12402e9c2f289b412c1966, entries=150, sequenceid=242, filesize=11.9 K 2024-12-04T14:57:11,535 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0fa368e0aeee714f048ec4ca8c747b4d in 921ms, sequenceid=242, compaction requested=true 2024-12-04T14:57:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-04T14:57:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-04T14:57:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:11,539 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:11,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-04T14:57:11,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/28701afd49f74341a54a82a5b99a99ec is 50, key is test_row_0/A:col10/1733324231429/Put/seqid=0 2024-12-04T14:57:11,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-04T14:57:11,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2290 sec 2024-12-04T14:57:11,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.2510 sec 2024-12-04T14:57:11,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742082_1258 (size=14541) 2024-12-04T14:57:11,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324291617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324291625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324291625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324291634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324291735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324291744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324291745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324291745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324291945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324291955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324291956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324291956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:11,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/28701afd49f74341a54a82a5b99a99ec 2024-12-04T14:57:12,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/31226912a61c43bc92f30ca1a612ded4 is 50, key is test_row_0/B:col10/1733324231429/Put/seqid=0 2024-12-04T14:57:12,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742083_1259 (size=12151) 2024-12-04T14:57:12,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/31226912a61c43bc92f30ca1a612ded4 2024-12-04T14:57:12,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/bf5447f9e41843699a6e4aabcb964839 is 50, key is test_row_0/C:col10/1733324231429/Put/seqid=0 2024-12-04T14:57:12,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742084_1260 (size=12151) 2024-12-04T14:57:12,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/bf5447f9e41843699a6e4aabcb964839 2024-12-04T14:57:12,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/28701afd49f74341a54a82a5b99a99ec as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/28701afd49f74341a54a82a5b99a99ec 2024-12-04T14:57:12,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/28701afd49f74341a54a82a5b99a99ec, entries=200, sequenceid=255, filesize=14.2 K 2024-12-04T14:57:12,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/31226912a61c43bc92f30ca1a612ded4 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/31226912a61c43bc92f30ca1a612ded4 2024-12-04T14:57:12,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/31226912a61c43bc92f30ca1a612ded4, entries=150, sequenceid=255, filesize=11.9 K 2024-12-04T14:57:12,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/bf5447f9e41843699a6e4aabcb964839 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/bf5447f9e41843699a6e4aabcb964839 2024-12-04T14:57:12,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/bf5447f9e41843699a6e4aabcb964839, entries=150, sequenceid=255, filesize=11.9 K 2024-12-04T14:57:12,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 0fa368e0aeee714f048ec4ca8c747b4d in 539ms, sequenceid=255, compaction requested=true 2024-12-04T14:57:12,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:12,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:12,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:12,078 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:57:12,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:12,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:12,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:12,078 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:57:12,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:12,079 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56218 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:57:12,079 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/A is initiating minor compaction (all files) 2024-12-04T14:57:12,080 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/A in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:12,080 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f80c3c77ad1a4974a5431458d47de66d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/5a1fa1bdd8d94237a5073f69fe5cea84, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/44ae287002414758958d63be8788bfb3, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/28701afd49f74341a54a82a5b99a99ec] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=54.9 K 2024-12-04T14:57:12,080 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:57:12,080 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/B is initiating minor compaction (all files) 2024-12-04T14:57:12,080 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/B in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:12,080 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2817593ff8a14023ae0a3f424dedf93e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/98e6f755f0a34d4d86857b14b7d96115, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a3a6b16cdf9f4f9391fafc5c71efc03b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/31226912a61c43bc92f30ca1a612ded4] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=47.9 K 2024-12-04T14:57:12,080 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting f80c3c77ad1a4974a5431458d47de66d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324228536 2024-12-04T14:57:12,080 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2817593ff8a14023ae0a3f424dedf93e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324228536 2024-12-04T14:57:12,081 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a1fa1bdd8d94237a5073f69fe5cea84, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733324229196 2024-12-04T14:57:12,081 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e6f755f0a34d4d86857b14b7d96115, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733324229221 2024-12-04T14:57:12,081 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3a6b16cdf9f4f9391fafc5c71efc03b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733324229267 2024-12-04T14:57:12,081 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31226912a61c43bc92f30ca1a612ded4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324231428 2024-12-04T14:57:12,081 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ae287002414758958d63be8788bfb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733324229267 2024-12-04T14:57:12,082 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 28701afd49f74341a54a82a5b99a99ec, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324231418 2024-12-04T14:57:12,092 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:12,093 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/bbf6ef22658f43e29e218a3f9eba1bd7 is 50, key is test_row_0/B:col10/1733324231429/Put/seqid=0 2024-12-04T14:57:12,097 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:12,098 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/f922681b8b834b7eb0108c15d9ffb04f is 50, key is test_row_0/A:col10/1733324231429/Put/seqid=0 2024-12-04T14:57:12,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742086_1262 (size=12731) 2024-12-04T14:57:12,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742085_1261 (size=12731) 2024-12-04T14:57:12,154 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/bbf6ef22658f43e29e218a3f9eba1bd7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/bbf6ef22658f43e29e218a3f9eba1bd7 2024-12-04T14:57:12,159 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/B of 0fa368e0aeee714f048ec4ca8c747b4d into bbf6ef22658f43e29e218a3f9eba1bd7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:12,159 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:12,159 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/B, priority=12, startTime=1733324232078; duration=0sec 2024-12-04T14:57:12,160 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:12,160 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:B 2024-12-04T14:57:12,160 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:57:12,162 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:57:12,162 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/C is initiating minor compaction (all files) 2024-12-04T14:57:12,162 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/C in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:12,162 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/2edc3fcbe9644af4bde97c6dea715629, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/c924ae000ddb4182ad218db19915b276, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4bd98c3d7a12402e9c2f289b412c1966, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/bf5447f9e41843699a6e4aabcb964839] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=47.9 K 2024-12-04T14:57:12,163 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2edc3fcbe9644af4bde97c6dea715629, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733324228536 2024-12-04T14:57:12,163 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting c924ae000ddb4182ad218db19915b276, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733324229221 2024-12-04T14:57:12,163 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bd98c3d7a12402e9c2f289b412c1966, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733324229267 2024-12-04T14:57:12,163 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf5447f9e41843699a6e4aabcb964839, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324231428 2024-12-04T14:57:12,177 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:12,177 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/53b3fc0b485249daad3c73898fde6354 is 50, key is test_row_0/C:col10/1733324231429/Put/seqid=0 2024-12-04T14:57:12,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742087_1263 (size=12731) 2024-12-04T14:57:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:12,258 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:12,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:57:12,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/2b3e8341f4f44d6aa3adc25263182ca5 is 50, key is test_row_0/A:col10/1733324232256/Put/seqid=0 2024-12-04T14:57:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324292274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324292277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324292277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324292278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742088_1264 (size=14741) 2024-12-04T14:57:12,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/2b3e8341f4f44d6aa3adc25263182ca5 2024-12-04T14:57:12,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/39f323e231d54072a2a3ab7b198d26bc is 50, key is test_row_0/B:col10/1733324232256/Put/seqid=0 2024-12-04T14:57:12,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742089_1265 (size=12301) 2024-12-04T14:57:12,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/39f323e231d54072a2a3ab7b198d26bc 2024-12-04T14:57:12,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/7244de35be864e4585a05b63db5721f4 is 50, key is test_row_0/C:col10/1733324232256/Put/seqid=0 2024-12-04T14:57:12,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742090_1266 (size=12301) 2024-12-04T14:57:12,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/7244de35be864e4585a05b63db5721f4 2024-12-04T14:57:12,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/2b3e8341f4f44d6aa3adc25263182ca5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/2b3e8341f4f44d6aa3adc25263182ca5 2024-12-04T14:57:12,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/2b3e8341f4f44d6aa3adc25263182ca5, entries=200, sequenceid=281, filesize=14.4 K 2024-12-04T14:57:12,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/39f323e231d54072a2a3ab7b198d26bc as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/39f323e231d54072a2a3ab7b198d26bc 2024-12-04T14:57:12,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324292386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324292386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324292386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324292385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/39f323e231d54072a2a3ab7b198d26bc, entries=150, sequenceid=281, filesize=12.0 K 2024-12-04T14:57:12,414 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-04T14:57:12,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/7244de35be864e4585a05b63db5721f4 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/7244de35be864e4585a05b63db5721f4 2024-12-04T14:57:12,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/7244de35be864e4585a05b63db5721f4, entries=150, sequenceid=281, filesize=12.0 K 2024-12-04T14:57:12,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 0fa368e0aeee714f048ec4ca8c747b4d in 162ms, sequenceid=281, compaction requested=false 2024-12-04T14:57:12,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T14:57:12,455 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:12,456 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-04T14:57:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:57:12,457 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:12,458 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:12,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:12,536 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/f922681b8b834b7eb0108c15d9ffb04f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f922681b8b834b7eb0108c15d9ffb04f 2024-12-04T14:57:12,541 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/A of 0fa368e0aeee714f048ec4ca8c747b4d into f922681b8b834b7eb0108c15d9ffb04f(size=12.4 K), total size for store is 26.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:12,541 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:12,541 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/A, priority=12, startTime=1733324232078; duration=0sec 2024-12-04T14:57:12,541 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:12,541 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:A 2024-12-04T14:57:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:57:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:12,598 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:12,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-04T14:57:12,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/b5652b9b28ca4ee189970bd9ec3393df is 50, key is test_row_0/A:col10/1733324232597/Put/seqid=0 2024-12-04T14:57:12,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742091_1267 (size=17181) 2024-12-04T14:57:12,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T14:57:12,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:12,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:12,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:12,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:12,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:12,613 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/53b3fc0b485249daad3c73898fde6354 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/53b3fc0b485249daad3c73898fde6354 2024-12-04T14:57:12,617 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/C of 0fa368e0aeee714f048ec4ca8c747b4d into 53b3fc0b485249daad3c73898fde6354(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:12,617 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:12,617 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/C, priority=12, startTime=1733324232078; duration=0sec 2024-12-04T14:57:12,617 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:12,618 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:C 2024-12-04T14:57:12,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324292622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324292622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324292622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324292623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324292735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324292735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324292735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324292735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T14:57:12,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:12,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:12,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:12,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:12,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:57:12,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T14:57:12,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:12,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:12,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:12,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:12,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:12,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324292945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324292945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324292945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:12,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:12,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324292945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:13,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/b5652b9b28ca4ee189970bd9ec3393df 2024-12-04T14:57:13,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/5097c786e3904c32a17035a9df8b31f7 is 50, key is test_row_0/B:col10/1733324232597/Put/seqid=0 2024-12-04T14:57:13,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742092_1268 (size=12301) 2024-12-04T14:57:13,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T14:57:13,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:13,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:57:13,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T14:57:13,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:13,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324293255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:13,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324293255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:13,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324293255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324293255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:13,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T14:57:13,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:13,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/5097c786e3904c32a17035a9df8b31f7 2024-12-04T14:57:13,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/298c403940304eb394a27053fc54882d is 50, key is test_row_0/C:col10/1733324232597/Put/seqid=0 2024-12-04T14:57:13,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742093_1269 (size=12301) 2024-12-04T14:57:13,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T14:57:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:57:13,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T14:57:13,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:13,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324293764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:13,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324293765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:13,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324293765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:13,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324293765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:13,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35756 deadline: 1733324293785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:13,786 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0fa368e0aeee714f048ec4ca8c747b4d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:13,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/298c403940304eb394a27053fc54882d 2024-12-04T14:57:13,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/b5652b9b28ca4ee189970bd9ec3393df as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b5652b9b28ca4ee189970bd9ec3393df 2024-12-04T14:57:13,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T14:57:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:13,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b5652b9b28ca4ee189970bd9ec3393df, entries=250, sequenceid=297, filesize=16.8 K 2024-12-04T14:57:13,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/5097c786e3904c32a17035a9df8b31f7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/5097c786e3904c32a17035a9df8b31f7 2024-12-04T14:57:13,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/5097c786e3904c32a17035a9df8b31f7, entries=150, sequenceid=297, filesize=12.0 K 2024-12-04T14:57:13,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/298c403940304eb394a27053fc54882d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/298c403940304eb394a27053fc54882d 2024-12-04T14:57:13,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/298c403940304eb394a27053fc54882d, entries=150, sequenceid=297, filesize=12.0 K 2024-12-04T14:57:13,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0fa368e0aeee714f048ec4ca8c747b4d in 1249ms, sequenceid=297, compaction requested=true 2024-12-04T14:57:13,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:13,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:13,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:13,847 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:13,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:13,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:13,847 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:13,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:13,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:13,848 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:13,848 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:13,848 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/A is initiating minor compaction (all files) 2024-12-04T14:57:13,848 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/B is initiating minor compaction (all files) 2024-12-04T14:57:13,848 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/B in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,848 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/A in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,848 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f922681b8b834b7eb0108c15d9ffb04f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/2b3e8341f4f44d6aa3adc25263182ca5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b5652b9b28ca4ee189970bd9ec3393df] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=43.6 K 2024-12-04T14:57:13,848 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/bbf6ef22658f43e29e218a3f9eba1bd7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/39f323e231d54072a2a3ab7b198d26bc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/5097c786e3904c32a17035a9df8b31f7] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=36.5 K 2024-12-04T14:57:13,848 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting bbf6ef22658f43e29e218a3f9eba1bd7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324231428 2024-12-04T14:57:13,848 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting f922681b8b834b7eb0108c15d9ffb04f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324231428 2024-12-04T14:57:13,849 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b3e8341f4f44d6aa3adc25263182ca5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733324231615 2024-12-04T14:57:13,849 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 39f323e231d54072a2a3ab7b198d26bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733324231615 2024-12-04T14:57:13,849 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5652b9b28ca4ee189970bd9ec3393df, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324232275 2024-12-04T14:57:13,849 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 5097c786e3904c32a17035a9df8b31f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324232275 2024-12-04T14:57:13,854 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:13,855 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/675485d0b8004bb497ddc2d5a8c057e8 is 50, key is test_row_0/A:col10/1733324232597/Put/seqid=0 2024-12-04T14:57:13,855 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:13,855 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/a970aa0fe5774e22acd2218d6e20b9e2 is 50, key is test_row_0/B:col10/1733324232597/Put/seqid=0 2024-12-04T14:57:13,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742095_1271 (size=12983) 2024-12-04T14:57:13,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742094_1270 (size=12983) 2024-12-04T14:57:13,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T14:57:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:13,986 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-04T14:57:13,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/1b4921463ffd4c8993fbddb8e0232f09 is 50, key is test_row_0/A:col10/1733324232621/Put/seqid=0 2024-12-04T14:57:13,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742096_1272 (size=12301) 2024-12-04T14:57:13,999 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/1b4921463ffd4c8993fbddb8e0232f09 2024-12-04T14:57:14,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/ee3e22f8194747f1b4b1005ed95a9321 is 50, key is test_row_0/B:col10/1733324232621/Put/seqid=0 2024-12-04T14:57:14,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742097_1273 (size=12301) 2024-12-04T14:57:14,262 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/a970aa0fe5774e22acd2218d6e20b9e2 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a970aa0fe5774e22acd2218d6e20b9e2 2024-12-04T14:57:14,263 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/675485d0b8004bb497ddc2d5a8c057e8 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/675485d0b8004bb497ddc2d5a8c057e8 2024-12-04T14:57:14,267 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/B of 0fa368e0aeee714f048ec4ca8c747b4d into a970aa0fe5774e22acd2218d6e20b9e2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:14,267 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/A of 0fa368e0aeee714f048ec4ca8c747b4d into 675485d0b8004bb497ddc2d5a8c057e8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:14,267 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:14,267 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:14,267 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/A, priority=13, startTime=1733324233847; duration=0sec 2024-12-04T14:57:14,267 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/B, priority=13, startTime=1733324233847; duration=0sec 2024-12-04T14:57:14,267 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:14,267 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:A 2024-12-04T14:57:14,267 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:14,267 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:B 2024-12-04T14:57:14,267 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:14,268 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:14,268 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/C is initiating minor compaction (all files) 2024-12-04T14:57:14,268 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/C in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:14,268 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/53b3fc0b485249daad3c73898fde6354, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/7244de35be864e4585a05b63db5721f4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/298c403940304eb394a27053fc54882d] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=36.5 K 2024-12-04T14:57:14,268 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53b3fc0b485249daad3c73898fde6354, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324231428 2024-12-04T14:57:14,269 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7244de35be864e4585a05b63db5721f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733324231615 2024-12-04T14:57:14,269 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 298c403940304eb394a27053fc54882d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324232275 2024-12-04T14:57:14,275 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#C#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:14,276 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/61dc8ae25c7b453bb28818636447e08c is 50, key is test_row_0/C:col10/1733324232597/Put/seqid=0 2024-12-04T14:57:14,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742098_1274 (size=12983) 2024-12-04T14:57:14,290 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/61dc8ae25c7b453bb28818636447e08c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/61dc8ae25c7b453bb28818636447e08c 2024-12-04T14:57:14,296 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/C of 0fa368e0aeee714f048ec4ca8c747b4d into 61dc8ae25c7b453bb28818636447e08c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:14,296 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:14,296 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/C, priority=13, startTime=1733324233847; duration=0sec 2024-12-04T14:57:14,296 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:14,296 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:C 2024-12-04T14:57:14,421 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/ee3e22f8194747f1b4b1005ed95a9321 2024-12-04T14:57:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/16e658ad308e4cfe93d1a52ffcdabc37 is 50, key is test_row_0/C:col10/1733324232621/Put/seqid=0 2024-12-04T14:57:14,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742099_1275 (size=12301) 2024-12-04T14:57:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:57:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:14,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324294786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324294787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:14,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324294788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324294789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:14,851 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/16e658ad308e4cfe93d1a52ffcdabc37 2024-12-04T14:57:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/1b4921463ffd4c8993fbddb8e0232f09 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1b4921463ffd4c8993fbddb8e0232f09 2024-12-04T14:57:14,859 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1b4921463ffd4c8993fbddb8e0232f09, entries=150, sequenceid=319, filesize=12.0 K 2024-12-04T14:57:14,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/ee3e22f8194747f1b4b1005ed95a9321 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ee3e22f8194747f1b4b1005ed95a9321 2024-12-04T14:57:14,863 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ee3e22f8194747f1b4b1005ed95a9321, entries=150, sequenceid=319, filesize=12.0 K 2024-12-04T14:57:14,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/16e658ad308e4cfe93d1a52ffcdabc37 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/16e658ad308e4cfe93d1a52ffcdabc37 2024-12-04T14:57:14,868 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/16e658ad308e4cfe93d1a52ffcdabc37, entries=150, sequenceid=319, filesize=12.0 K 2024-12-04T14:57:14,869 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 0fa368e0aeee714f048ec4ca8c747b4d in 883ms, sequenceid=319, compaction requested=false 2024-12-04T14:57:14,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:14,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:14,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-04T14:57:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-04T14:57:14,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-04T14:57:14,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4120 sec 2024-12-04T14:57:14,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.4160 sec 2024-12-04T14:57:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:14,897 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:14,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-04T14:57:14,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/8066309eaf174645a321c55e8d89579e is 50, key is test_row_0/A:col10/1733324234897/Put/seqid=0 2024-12-04T14:57:14,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742100_1276 (size=14741) 2024-12-04T14:57:14,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/8066309eaf174645a321c55e8d89579e 2024-12-04T14:57:14,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/45b8a716ecb74ca5a955c46784c5c5e9 is 50, key is test_row_0/B:col10/1733324234897/Put/seqid=0 2024-12-04T14:57:14,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324294913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:14,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324294913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:14,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742101_1277 (size=12301) 2024-12-04T14:57:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/45b8a716ecb74ca5a955c46784c5c5e9 2024-12-04T14:57:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:14,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324294917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:14,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324294918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:14,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/fb977cbf2a65453cace4349df9871329 is 50, key is test_row_0/C:col10/1733324234897/Put/seqid=0 2024-12-04T14:57:14,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742102_1278 (size=12301) 2024-12-04T14:57:14,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/fb977cbf2a65453cace4349df9871329 2024-12-04T14:57:14,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/8066309eaf174645a321c55e8d89579e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/8066309eaf174645a321c55e8d89579e 2024-12-04T14:57:14,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/8066309eaf174645a321c55e8d89579e, entries=200, sequenceid=339, filesize=14.4 K 2024-12-04T14:57:14,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/45b8a716ecb74ca5a955c46784c5c5e9 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/45b8a716ecb74ca5a955c46784c5c5e9 2024-12-04T14:57:14,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/45b8a716ecb74ca5a955c46784c5c5e9, entries=150, sequenceid=339, filesize=12.0 K 2024-12-04T14:57:14,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/fb977cbf2a65453cace4349df9871329 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/fb977cbf2a65453cace4349df9871329 2024-12-04T14:57:14,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/fb977cbf2a65453cace4349df9871329, entries=150, sequenceid=339, filesize=12.0 K 2024-12-04T14:57:14,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 0fa368e0aeee714f048ec4ca8c747b4d in 52ms, sequenceid=339, compaction requested=true 2024-12-04T14:57:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:14,950 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:14,950 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:14,951 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:14,951 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/B is initiating minor compaction (all files) 2024-12-04T14:57:14,951 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/B in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:14,951 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a970aa0fe5774e22acd2218d6e20b9e2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ee3e22f8194747f1b4b1005ed95a9321, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/45b8a716ecb74ca5a955c46784c5c5e9] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=36.7 K 2024-12-04T14:57:14,951 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:14,951 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/A is initiating minor compaction (all files) 2024-12-04T14:57:14,951 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/A in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:14,951 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/675485d0b8004bb497ddc2d5a8c057e8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1b4921463ffd4c8993fbddb8e0232f09, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/8066309eaf174645a321c55e8d89579e] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=39.1 K 2024-12-04T14:57:14,952 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 675485d0b8004bb497ddc2d5a8c057e8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324232275 2024-12-04T14:57:14,952 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting a970aa0fe5774e22acd2218d6e20b9e2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324232275 2024-12-04T14:57:14,952 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3e22f8194747f1b4b1005ed95a9321, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733324232621 2024-12-04T14:57:14,952 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b4921463ffd4c8993fbddb8e0232f09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733324232621 2024-12-04T14:57:14,952 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b8a716ecb74ca5a955c46784c5c5e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733324234787 2024-12-04T14:57:14,952 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8066309eaf174645a321c55e8d89579e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733324234784 2024-12-04T14:57:14,959 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:14,959 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#A#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:14,959 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/d44ef1b2c9924ba29e0050b443bcacfd is 50, key is test_row_0/B:col10/1733324234897/Put/seqid=0 2024-12-04T14:57:14,960 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/748c92b6efae49fe84db7a4cb15aaf22 is 50, key is test_row_0/A:col10/1733324234897/Put/seqid=0 2024-12-04T14:57:14,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742104_1280 (size=13085) 2024-12-04T14:57:14,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742103_1279 (size=13085) 2024-12-04T14:57:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:15,027 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:15,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-04T14:57:15,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/3fde4a00e3544970b344507eb75f9040 is 50, key is test_row_0/A:col10/1733324235026/Put/seqid=0 2024-12-04T14:57:15,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742105_1281 (size=12301) 2024-12-04T14:57:15,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/3fde4a00e3544970b344507eb75f9040 2024-12-04T14:57:15,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324295039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/dfd69ba8f2cf4133a0a2663748503688 is 50, key is test_row_0/B:col10/1733324235026/Put/seqid=0 2024-12-04T14:57:15,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324295041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324295042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324295043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742106_1282 (size=12301) 2024-12-04T14:57:15,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324295145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324295155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324295155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324295155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324295355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324295365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324295365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324295365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,376 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/748c92b6efae49fe84db7a4cb15aaf22 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/748c92b6efae49fe84db7a4cb15aaf22 2024-12-04T14:57:15,381 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/A of 0fa368e0aeee714f048ec4ca8c747b4d into 748c92b6efae49fe84db7a4cb15aaf22(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:15,381 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:15,381 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/A, priority=13, startTime=1733324234950; duration=0sec 2024-12-04T14:57:15,381 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:15,381 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:A 2024-12-04T14:57:15,381 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:15,382 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:15,382 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/C is initiating minor compaction (all files) 2024-12-04T14:57:15,382 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/C in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:15,382 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/61dc8ae25c7b453bb28818636447e08c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/16e658ad308e4cfe93d1a52ffcdabc37, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/fb977cbf2a65453cace4349df9871329] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=36.7 K 2024-12-04T14:57:15,383 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61dc8ae25c7b453bb28818636447e08c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733324232275 2024-12-04T14:57:15,383 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16e658ad308e4cfe93d1a52ffcdabc37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733324232621 2024-12-04T14:57:15,383 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb977cbf2a65453cace4349df9871329, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733324234787 2024-12-04T14:57:15,389 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/d44ef1b2c9924ba29e0050b443bcacfd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/d44ef1b2c9924ba29e0050b443bcacfd 2024-12-04T14:57:15,391 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#C#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:15,392 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/8d49502565594954ab2e817f34d180d9 is 50, key is test_row_0/C:col10/1733324234897/Put/seqid=0 2024-12-04T14:57:15,394 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/B of 0fa368e0aeee714f048ec4ca8c747b4d into d44ef1b2c9924ba29e0050b443bcacfd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:15,394 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:15,394 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/B, priority=13, startTime=1733324234950; duration=0sec 2024-12-04T14:57:15,394 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:15,394 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:B 2024-12-04T14:57:15,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742107_1283 (size=13085) 2024-12-04T14:57:15,404 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/8d49502565594954ab2e817f34d180d9 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/8d49502565594954ab2e817f34d180d9 2024-12-04T14:57:15,409 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/C of 0fa368e0aeee714f048ec4ca8c747b4d into 8d49502565594954ab2e817f34d180d9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:15,409 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:15,409 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/C, priority=13, startTime=1733324234950; duration=0sec 2024-12-04T14:57:15,409 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:15,409 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:C 2024-12-04T14:57:15,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/dfd69ba8f2cf4133a0a2663748503688 2024-12-04T14:57:15,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/1418d2fb969d418d90ebc919db44ed95 is 50, key is test_row_0/C:col10/1733324235026/Put/seqid=0 2024-12-04T14:57:15,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742108_1284 (size=12301) 2024-12-04T14:57:15,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324295664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324295675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324295675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324295675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:15,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/1418d2fb969d418d90ebc919db44ed95 2024-12-04T14:57:15,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/3fde4a00e3544970b344507eb75f9040 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3fde4a00e3544970b344507eb75f9040 2024-12-04T14:57:15,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3fde4a00e3544970b344507eb75f9040, entries=150, sequenceid=361, filesize=12.0 K 2024-12-04T14:57:15,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/dfd69ba8f2cf4133a0a2663748503688 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dfd69ba8f2cf4133a0a2663748503688 2024-12-04T14:57:15,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dfd69ba8f2cf4133a0a2663748503688, entries=150, sequenceid=361, filesize=12.0 K 2024-12-04T14:57:15,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/1418d2fb969d418d90ebc919db44ed95 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/1418d2fb969d418d90ebc919db44ed95 2024-12-04T14:57:15,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/1418d2fb969d418d90ebc919db44ed95, entries=150, sequenceid=361, filesize=12.0 K 2024-12-04T14:57:15,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0fa368e0aeee714f048ec4ca8c747b4d in 849ms, sequenceid=361, compaction requested=false 2024-12-04T14:57:15,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:16,176 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:16,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-04T14:57:16,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/7709c9868b1a4403bd9c224fe570b9a0 is 50, key is test_row_0/A:col10/1733324235036/Put/seqid=0 2024-12-04T14:57:16,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742109_1285 (size=14741) 2024-12-04T14:57:16,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324296201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324296204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324296205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324296205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324296315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324296315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324296315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324296315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324296524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324296525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324296525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324296525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/7709c9868b1a4403bd9c224fe570b9a0 2024-12-04T14:57:16,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/36c031e268564ce3ae70a7c47aa9a842 is 50, key is test_row_0/B:col10/1733324235036/Put/seqid=0 2024-12-04T14:57:16,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742110_1286 (size=12301) 2024-12-04T14:57:16,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/36c031e268564ce3ae70a7c47aa9a842 2024-12-04T14:57:16,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/b76352754bf94b84a8e176aaaea41c50 is 50, key is test_row_0/C:col10/1733324235036/Put/seqid=0 2024-12-04T14:57:16,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742111_1287 (size=12301) 2024-12-04T14:57:16,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/b76352754bf94b84a8e176aaaea41c50 2024-12-04T14:57:16,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/7709c9868b1a4403bd9c224fe570b9a0 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/7709c9868b1a4403bd9c224fe570b9a0 2024-12-04T14:57:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T14:57:16,625 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:16,626 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-04T14:57:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T14:57:16,628 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:16,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/7709c9868b1a4403bd9c224fe570b9a0, entries=200, sequenceid=379, filesize=14.4 K 2024-12-04T14:57:16,630 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:16,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/36c031e268564ce3ae70a7c47aa9a842 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/36c031e268564ce3ae70a7c47aa9a842 2024-12-04T14:57:16,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/36c031e268564ce3ae70a7c47aa9a842, entries=150, sequenceid=379, filesize=12.0 K 2024-12-04T14:57:16,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/b76352754bf94b84a8e176aaaea41c50 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/b76352754bf94b84a8e176aaaea41c50 2024-12-04T14:57:16,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/b76352754bf94b84a8e176aaaea41c50, entries=150, sequenceid=379, filesize=12.0 K 2024-12-04T14:57:16,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 0fa368e0aeee714f048ec4ca8c747b4d in 465ms, sequenceid=379, compaction requested=true 2024-12-04T14:57:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:16,641 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T14:57:16,641 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:16,642 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:16,642 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/A is initiating minor compaction (all files) 2024-12-04T14:57:16,642 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:16,642 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/A in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:16,642 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/B is initiating minor compaction (all files) 2024-12-04T14:57:16,642 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/748c92b6efae49fe84db7a4cb15aaf22, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3fde4a00e3544970b344507eb75f9040, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/7709c9868b1a4403bd9c224fe570b9a0] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=39.2 K 2024-12-04T14:57:16,642 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/B in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:16,642 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/d44ef1b2c9924ba29e0050b443bcacfd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dfd69ba8f2cf4133a0a2663748503688, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/36c031e268564ce3ae70a7c47aa9a842] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=36.8 K 2024-12-04T14:57:16,643 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 748c92b6efae49fe84db7a4cb15aaf22, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733324234787 2024-12-04T14:57:16,643 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting d44ef1b2c9924ba29e0050b443bcacfd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733324234787 2024-12-04T14:57:16,643 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fde4a00e3544970b344507eb75f9040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733324234910 2024-12-04T14:57:16,643 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting dfd69ba8f2cf4133a0a2663748503688, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733324234910 2024-12-04T14:57:16,643 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7709c9868b1a4403bd9c224fe570b9a0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733324235036 2024-12-04T14:57:16,643 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 36c031e268564ce3ae70a7c47aa9a842, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733324235036 2024-12-04T14:57:16,650 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:16,650 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/cd31ce750c6e4fa5ad95c5c43749de79 is 50, key is test_row_0/B:col10/1733324235036/Put/seqid=0 2024-12-04T14:57:16,653 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:16,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742112_1288 (size=13187) 2024-12-04T14:57:16,654 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/aeea0405c311411d9a21229be343cbcc is 50, key is test_row_0/A:col10/1733324235036/Put/seqid=0 2024-12-04T14:57:16,662 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/cd31ce750c6e4fa5ad95c5c43749de79 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/cd31ce750c6e4fa5ad95c5c43749de79 2024-12-04T14:57:16,667 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/B of 0fa368e0aeee714f048ec4ca8c747b4d into cd31ce750c6e4fa5ad95c5c43749de79(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:16,668 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:16,668 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/B, priority=13, startTime=1733324236641; duration=0sec 2024-12-04T14:57:16,668 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:16,668 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:B 2024-12-04T14:57:16,668 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:16,669 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:16,669 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/C is initiating minor compaction (all files) 2024-12-04T14:57:16,669 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/C in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:16,669 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/8d49502565594954ab2e817f34d180d9, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/1418d2fb969d418d90ebc919db44ed95, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/b76352754bf94b84a8e176aaaea41c50] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=36.8 K 2024-12-04T14:57:16,670 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d49502565594954ab2e817f34d180d9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733324234787 2024-12-04T14:57:16,670 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 1418d2fb969d418d90ebc919db44ed95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733324234910 2024-12-04T14:57:16,671 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b76352754bf94b84a8e176aaaea41c50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733324235036 2024-12-04T14:57:16,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742113_1289 (size=13187) 2024-12-04T14:57:16,678 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/aeea0405c311411d9a21229be343cbcc as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/aeea0405c311411d9a21229be343cbcc 2024-12-04T14:57:16,682 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:16,683 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/5ecdcca69e804a47a3784e926951bb70 is 50, key is test_row_0/C:col10/1733324235036/Put/seqid=0 2024-12-04T14:57:16,683 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/A of 0fa368e0aeee714f048ec4ca8c747b4d into aeea0405c311411d9a21229be343cbcc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:16,683 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:16,683 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/A, priority=13, startTime=1733324236641; duration=0sec 2024-12-04T14:57:16,683 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:16,683 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:A 2024-12-04T14:57:16,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742114_1290 (size=13187) 2024-12-04T14:57:16,691 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/5ecdcca69e804a47a3784e926951bb70 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ecdcca69e804a47a3784e926951bb70 2024-12-04T14:57:16,696 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/C of 0fa368e0aeee714f048ec4ca8c747b4d into 5ecdcca69e804a47a3784e926951bb70(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:16,696 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:16,696 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/C, priority=13, startTime=1733324236641; duration=0sec 2024-12-04T14:57:16,697 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:16,697 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:C 2024-12-04T14:57:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T14:57:16,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T14:57:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:16,783 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-04T14:57:16,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/b7337ba92b8149a48333225973bc6e60 is 50, key is test_row_0/A:col10/1733324236204/Put/seqid=0 2024-12-04T14:57:16,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742115_1291 (size=12301) 2024-12-04T14:57:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:16,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. as already flushing 2024-12-04T14:57:16,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324296842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324296843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324296844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324296846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T14:57:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324296955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324296955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324296955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324296955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324297165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324297165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324297165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324297165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,191 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/b7337ba92b8149a48333225973bc6e60 2024-12-04T14:57:17,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/0055091b35e14877ba3081b42af5d851 is 50, key is test_row_0/B:col10/1733324236204/Put/seqid=0 2024-12-04T14:57:17,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742116_1292 (size=12301) 2024-12-04T14:57:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T14:57:17,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324297474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324297474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324297475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324297475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,603 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/0055091b35e14877ba3081b42af5d851 2024-12-04T14:57:17,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/f08f4f1c38d949239ea6cdf8e41fb026 is 50, key is test_row_0/C:col10/1733324236204/Put/seqid=0 2024-12-04T14:57:17,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742117_1293 (size=12301) 2024-12-04T14:57:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T14:57:17,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35712 deadline: 1733324297985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35764 deadline: 1733324297985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35714 deadline: 1733324297985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:17,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1733324297985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:18,042 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/f08f4f1c38d949239ea6cdf8e41fb026 2024-12-04T14:57:18,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/b7337ba92b8149a48333225973bc6e60 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b7337ba92b8149a48333225973bc6e60 2024-12-04T14:57:18,049 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b7337ba92b8149a48333225973bc6e60, entries=150, sequenceid=404, filesize=12.0 K 2024-12-04T14:57:18,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/0055091b35e14877ba3081b42af5d851 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0055091b35e14877ba3081b42af5d851 2024-12-04T14:57:18,054 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0055091b35e14877ba3081b42af5d851, entries=150, sequenceid=404, filesize=12.0 K 2024-12-04T14:57:18,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/f08f4f1c38d949239ea6cdf8e41fb026 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f08f4f1c38d949239ea6cdf8e41fb026 2024-12-04T14:57:18,058 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f08f4f1c38d949239ea6cdf8e41fb026, entries=150, sequenceid=404, filesize=12.0 K 2024-12-04T14:57:18,059 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0fa368e0aeee714f048ec4ca8c747b4d in 1276ms, sequenceid=404, compaction requested=false 2024-12-04T14:57:18,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:18,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:18,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-04T14:57:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-04T14:57:18,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-04T14:57:18,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-12-04T14:57:18,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4360 sec 2024-12-04T14:57:18,680 INFO [Thread-1013 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1013. 2024-12-04T14:57:18,680 DEBUG [Thread-1013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:18,680 DEBUG [Thread-1013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,680 DEBUG [Thread-1013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:18,681 INFO [Thread-1009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1009. 2024-12-04T14:57:18,681 DEBUG [Thread-1009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:18,681 DEBUG [Thread-1009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,681 DEBUG [Thread-1009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:18,681 INFO [Thread-1007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1007. 2024-12-04T14:57:18,681 DEBUG [Thread-1007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:18,681 DEBUG [Thread-1007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,681 DEBUG [Thread-1007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:18,682 INFO [Thread-1011 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1011. 2024-12-04T14:57:18,682 DEBUG [Thread-1011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:18,682 DEBUG [Thread-1011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,682 DEBUG [Thread-1011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:18,682 INFO [Thread-1005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1005. 2024-12-04T14:57:18,682 DEBUG [Thread-1005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:18,682 DEBUG [Thread-1005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,682 DEBUG [Thread-1005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T14:57:18,775 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:18,997 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:18,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-04T14:57:18,998 INFO [Thread-994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-994. 2024-12-04T14:57:18,998 INFO [Thread-996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-996. 2024-12-04T14:57:18,998 INFO [Thread-998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-998. 2024-12-04T14:57:18,998 DEBUG [Thread-994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:18,998 DEBUG [Thread-994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,998 DEBUG [Thread-998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:18,998 DEBUG [Thread-996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:18,998 DEBUG [Thread-998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,998 DEBUG [Thread-996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,998 DEBUG [Thread-994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:18,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:18,998 DEBUG [Thread-996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,998 DEBUG [Thread-998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:18,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:18,999 INFO [Thread-1000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1000. 2024-12-04T14:57:19,000 DEBUG [Thread-1000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:19,000 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:19,000 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:19,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:19,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/af6093df62af4e13892c19117330a6bd is 50, key is test_row_0/A:col10/1733324236843/Put/seqid=0 2024-12-04T14:57:19,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742118_1294 (size=12301) 2024-12-04T14:57:19,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/af6093df62af4e13892c19117330a6bd 2024-12-04T14:57:19,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/26f05b241e574e48ab2a7169c7327c68 is 50, key is test_row_0/B:col10/1733324236843/Put/seqid=0 2024-12-04T14:57:19,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742119_1295 (size=12301) 2024-12-04T14:57:19,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/26f05b241e574e48ab2a7169c7327c68 2024-12-04T14:57:19,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/911e7c801d49453fa41db21d579993ad is 50, key is test_row_0/C:col10/1733324236843/Put/seqid=0 2024-12-04T14:57:19,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742120_1296 (size=12301) 2024-12-04T14:57:20,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/911e7c801d49453fa41db21d579993ad 2024-12-04T14:57:20,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/af6093df62af4e13892c19117330a6bd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af6093df62af4e13892c19117330a6bd 2024-12-04T14:57:20,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af6093df62af4e13892c19117330a6bd, entries=150, sequenceid=422, filesize=12.0 K 2024-12-04T14:57:20,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/26f05b241e574e48ab2a7169c7327c68 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/26f05b241e574e48ab2a7169c7327c68 2024-12-04T14:57:20,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/26f05b241e574e48ab2a7169c7327c68, entries=150, sequenceid=422, filesize=12.0 K 2024-12-04T14:57:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/911e7c801d49453fa41db21d579993ad as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/911e7c801d49453fa41db21d579993ad 2024-12-04T14:57:20,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/911e7c801d49453fa41db21d579993ad, entries=150, sequenceid=422, filesize=12.0 K 2024-12-04T14:57:20,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 0fa368e0aeee714f048ec4ca8c747b4d in 1282ms, sequenceid=422, compaction requested=true 2024-12-04T14:57:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:20,279 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fa368e0aeee714f048ec4ca8c747b4d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:20,279 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:20,280 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:20,280 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:20,281 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/B is initiating minor compaction (all files) 2024-12-04T14:57:20,281 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/A is initiating minor compaction (all files) 2024-12-04T14:57:20,281 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/B in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:20,281 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/A in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:20,281 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/cd31ce750c6e4fa5ad95c5c43749de79, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0055091b35e14877ba3081b42af5d851, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/26f05b241e574e48ab2a7169c7327c68] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=36.9 K 2024-12-04T14:57:20,281 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/aeea0405c311411d9a21229be343cbcc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b7337ba92b8149a48333225973bc6e60, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af6093df62af4e13892c19117330a6bd] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=36.9 K 2024-12-04T14:57:20,281 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeea0405c311411d9a21229be343cbcc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733324235036 2024-12-04T14:57:20,281 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting cd31ce750c6e4fa5ad95c5c43749de79, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733324235036 2024-12-04T14:57:20,282 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7337ba92b8149a48333225973bc6e60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1733324236192 2024-12-04T14:57:20,282 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 0055091b35e14877ba3081b42af5d851, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1733324236192 2024-12-04T14:57:20,282 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 26f05b241e574e48ab2a7169c7327c68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733324236843 2024-12-04T14:57:20,282 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting af6093df62af4e13892c19117330a6bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733324236843 2024-12-04T14:57:20,288 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:20,288 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:20,288 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/1f81d16a481c456ca1a1263fc532b611 is 50, key is test_row_0/A:col10/1733324236843/Put/seqid=0 2024-12-04T14:57:20,288 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/ecb4c7a2c1474d4bb51cfb9c5a4a030b is 50, key is test_row_0/B:col10/1733324236843/Put/seqid=0 2024-12-04T14:57:20,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742122_1298 (size=13289) 2024-12-04T14:57:20,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742121_1297 (size=13289) 2024-12-04T14:57:20,699 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/ecb4c7a2c1474d4bb51cfb9c5a4a030b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ecb4c7a2c1474d4bb51cfb9c5a4a030b 2024-12-04T14:57:20,699 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/1f81d16a481c456ca1a1263fc532b611 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1f81d16a481c456ca1a1263fc532b611 2024-12-04T14:57:20,703 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/B of 0fa368e0aeee714f048ec4ca8c747b4d into ecb4c7a2c1474d4bb51cfb9c5a4a030b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:20,703 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/A of 0fa368e0aeee714f048ec4ca8c747b4d into 1f81d16a481c456ca1a1263fc532b611(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:20,703 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:20,703 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:20,703 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/A, priority=13, startTime=1733324240279; duration=0sec 2024-12-04T14:57:20,703 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/B, priority=13, startTime=1733324240279; duration=0sec 2024-12-04T14:57:20,703 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:20,703 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:20,703 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:A 2024-12-04T14:57:20,703 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:B 2024-12-04T14:57:20,703 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:20,704 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:20,704 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0fa368e0aeee714f048ec4ca8c747b4d/C is initiating minor compaction (all files) 2024-12-04T14:57:20,704 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fa368e0aeee714f048ec4ca8c747b4d/C in TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:20,704 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ecdcca69e804a47a3784e926951bb70, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f08f4f1c38d949239ea6cdf8e41fb026, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/911e7c801d49453fa41db21d579993ad] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp, totalSize=36.9 K 2024-12-04T14:57:20,705 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ecdcca69e804a47a3784e926951bb70, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733324235036 2024-12-04T14:57:20,705 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting f08f4f1c38d949239ea6cdf8e41fb026, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1733324236192 2024-12-04T14:57:20,705 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 911e7c801d49453fa41db21d579993ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733324236843 2024-12-04T14:57:20,711 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fa368e0aeee714f048ec4ca8c747b4d#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:20,712 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/8cdf9873c0f24c878902b080234afd4b is 50, key is test_row_0/C:col10/1733324236843/Put/seqid=0 2024-12-04T14:57:20,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742123_1299 (size=13289) 2024-12-04T14:57:21,128 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/8cdf9873c0f24c878902b080234afd4b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/8cdf9873c0f24c878902b080234afd4b 2024-12-04T14:57:21,134 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fa368e0aeee714f048ec4ca8c747b4d/C of 0fa368e0aeee714f048ec4ca8c747b4d into 8cdf9873c0f24c878902b080234afd4b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:21,134 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fa368e0aeee714f048ec4ca8c747b4d: 2024-12-04T14:57:21,134 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., storeName=0fa368e0aeee714f048ec4ca8c747b4d/C, priority=13, startTime=1733324240279; duration=0sec 2024-12-04T14:57:21,134 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:21,134 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fa368e0aeee714f048ec4ca8c747b4d:C 2024-12-04T14:57:23,847 INFO [Thread-1002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1002. 2024-12-04T14:57:23,847 DEBUG [Thread-1002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:23,847 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:23,848 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:23,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T14:57:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-04T14:57:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-04T14:57:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-12-04T14:57:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-12-04T14:57:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-12-04T14:57:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T14:57:23,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5282 2024-12-04T14:57:23,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5278 2024-12-04T14:57:23,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5306 2024-12-04T14:57:23,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5290 2024-12-04T14:57:23,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5282 2024-12-04T14:57:23,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T14:57:23,849 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:57:23,849 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:57:23,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:23,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:23,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:23,851 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T14:57:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T14:57:23,856 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324243856"}]},"ts":"1733324243856"} 2024-12-04T14:57:23,858 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T14:57:23,858 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T14:57:23,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:57:23,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa368e0aeee714f048ec4ca8c747b4d, UNASSIGN}] 2024-12-04T14:57:23,861 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa368e0aeee714f048ec4ca8c747b4d, UNASSIGN 2024-12-04T14:57:23,862 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=0fa368e0aeee714f048ec4ca8c747b4d, regionState=CLOSING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:23,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa368e0aeee714f048ec4ca8c747b4d, UNASSIGN because future has completed 2024-12-04T14:57:23,863 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:57:23,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:57:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T14:57:24,017 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:24,017 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:57:24,017 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 0fa368e0aeee714f048ec4ca8c747b4d, disabling compactions & flushes 2024-12-04T14:57:24,017 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:24,017 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:24,017 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. after waiting 0 ms 2024-12-04T14:57:24,017 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:24,018 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 0fa368e0aeee714f048ec4ca8c747b4d 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-12-04T14:57:24,024 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/499a048387234162aa98f3205ce276ea is 50, key is test_row_0/A:col10/1733324243846/Put/seqid=0 2024-12-04T14:57:24,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742124_1300 (size=7415) 2024-12-04T14:57:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T14:57:24,430 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/499a048387234162aa98f3205ce276ea 2024-12-04T14:57:24,441 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/bd206f5828da4ec8a85ecc5727b7fa3a is 50, key is test_row_0/B:col10/1733324243846/Put/seqid=0 2024-12-04T14:57:24,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742125_1301 (size=7415) 2024-12-04T14:57:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T14:57:24,847 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/bd206f5828da4ec8a85ecc5727b7fa3a 2024-12-04T14:57:24,861 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/d587147640724d4c8491c5c73aeab32f is 50, key is test_row_0/C:col10/1733324243846/Put/seqid=0 2024-12-04T14:57:24,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742126_1302 (size=7415) 2024-12-04T14:57:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T14:57:25,379 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/d587147640724d4c8491c5c73aeab32f 2024-12-04T14:57:25,390 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/A/499a048387234162aa98f3205ce276ea as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/499a048387234162aa98f3205ce276ea 2024-12-04T14:57:25,396 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/499a048387234162aa98f3205ce276ea, entries=50, sequenceid=429, filesize=7.2 K 2024-12-04T14:57:25,397 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/B/bd206f5828da4ec8a85ecc5727b7fa3a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/bd206f5828da4ec8a85ecc5727b7fa3a 2024-12-04T14:57:25,402 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/bd206f5828da4ec8a85ecc5727b7fa3a, entries=50, sequenceid=429, filesize=7.2 K 2024-12-04T14:57:25,403 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/.tmp/C/d587147640724d4c8491c5c73aeab32f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/d587147640724d4c8491c5c73aeab32f 2024-12-04T14:57:25,407 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/d587147640724d4c8491c5c73aeab32f, entries=50, sequenceid=429, filesize=7.2 K 2024-12-04T14:57:25,407 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 0fa368e0aeee714f048ec4ca8c747b4d in 1389ms, sequenceid=429, compaction requested=false 2024-12-04T14:57:25,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1298c8a1afa44de2905c9e336f4ffc57, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af70a43683bb4afdb3255c3dca86634c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f601d3ffe0d548d4b3c778796703cb88, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1bc76e67f4f94e188f0bad1d6fc3516d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/a097c537a244446ea86167ce13ef191a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/eab325128a264888bcded4e8910d4ebd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/387aab508e8c4135baab0d417c4dba61, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3e0f51359a7740f8b24dd7481e6c71fe, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/25ea3705e9c74f6da4097ae5841c6b88, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/154a11dd3e88456587558ebccae6ff1d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/d2c6709dcc104a5a91405401802121ba, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/89b1d737a1b84a2ca50cd57ac381c8f8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f80c3c77ad1a4974a5431458d47de66d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/6180396ebbfe4d63badad91def6666cd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/5a1fa1bdd8d94237a5073f69fe5cea84, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/44ae287002414758958d63be8788bfb3, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/28701afd49f74341a54a82a5b99a99ec, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f922681b8b834b7eb0108c15d9ffb04f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/2b3e8341f4f44d6aa3adc25263182ca5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b5652b9b28ca4ee189970bd9ec3393df, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/675485d0b8004bb497ddc2d5a8c057e8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1b4921463ffd4c8993fbddb8e0232f09, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/8066309eaf174645a321c55e8d89579e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/748c92b6efae49fe84db7a4cb15aaf22, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3fde4a00e3544970b344507eb75f9040, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/7709c9868b1a4403bd9c224fe570b9a0, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/aeea0405c311411d9a21229be343cbcc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b7337ba92b8149a48333225973bc6e60, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af6093df62af4e13892c19117330a6bd] to archive 2024-12-04T14:57:25,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:57:25,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1298c8a1afa44de2905c9e336f4ffc57 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1298c8a1afa44de2905c9e336f4ffc57 2024-12-04T14:57:25,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af70a43683bb4afdb3255c3dca86634c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af70a43683bb4afdb3255c3dca86634c 2024-12-04T14:57:25,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f601d3ffe0d548d4b3c778796703cb88 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f601d3ffe0d548d4b3c778796703cb88 2024-12-04T14:57:25,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1bc76e67f4f94e188f0bad1d6fc3516d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1bc76e67f4f94e188f0bad1d6fc3516d 2024-12-04T14:57:25,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/a097c537a244446ea86167ce13ef191a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/a097c537a244446ea86167ce13ef191a 2024-12-04T14:57:25,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/eab325128a264888bcded4e8910d4ebd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/eab325128a264888bcded4e8910d4ebd 2024-12-04T14:57:25,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/387aab508e8c4135baab0d417c4dba61 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/387aab508e8c4135baab0d417c4dba61 2024-12-04T14:57:25,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3e0f51359a7740f8b24dd7481e6c71fe to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3e0f51359a7740f8b24dd7481e6c71fe 2024-12-04T14:57:25,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/25ea3705e9c74f6da4097ae5841c6b88 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/25ea3705e9c74f6da4097ae5841c6b88 2024-12-04T14:57:25,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/154a11dd3e88456587558ebccae6ff1d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/154a11dd3e88456587558ebccae6ff1d 2024-12-04T14:57:25,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/d2c6709dcc104a5a91405401802121ba to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/d2c6709dcc104a5a91405401802121ba 2024-12-04T14:57:25,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/89b1d737a1b84a2ca50cd57ac381c8f8 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/89b1d737a1b84a2ca50cd57ac381c8f8 2024-12-04T14:57:25,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f80c3c77ad1a4974a5431458d47de66d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f80c3c77ad1a4974a5431458d47de66d 2024-12-04T14:57:25,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/6180396ebbfe4d63badad91def6666cd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/6180396ebbfe4d63badad91def6666cd 2024-12-04T14:57:25,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/5a1fa1bdd8d94237a5073f69fe5cea84 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/5a1fa1bdd8d94237a5073f69fe5cea84 2024-12-04T14:57:25,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/44ae287002414758958d63be8788bfb3 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/44ae287002414758958d63be8788bfb3 2024-12-04T14:57:25,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/28701afd49f74341a54a82a5b99a99ec to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/28701afd49f74341a54a82a5b99a99ec 2024-12-04T14:57:25,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f922681b8b834b7eb0108c15d9ffb04f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/f922681b8b834b7eb0108c15d9ffb04f 2024-12-04T14:57:25,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/2b3e8341f4f44d6aa3adc25263182ca5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/2b3e8341f4f44d6aa3adc25263182ca5 2024-12-04T14:57:25,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b5652b9b28ca4ee189970bd9ec3393df to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b5652b9b28ca4ee189970bd9ec3393df 2024-12-04T14:57:25,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/675485d0b8004bb497ddc2d5a8c057e8 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/675485d0b8004bb497ddc2d5a8c057e8 2024-12-04T14:57:25,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1b4921463ffd4c8993fbddb8e0232f09 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1b4921463ffd4c8993fbddb8e0232f09 2024-12-04T14:57:25,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/8066309eaf174645a321c55e8d89579e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/8066309eaf174645a321c55e8d89579e 2024-12-04T14:57:25,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/748c92b6efae49fe84db7a4cb15aaf22 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/748c92b6efae49fe84db7a4cb15aaf22 2024-12-04T14:57:25,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3fde4a00e3544970b344507eb75f9040 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/3fde4a00e3544970b344507eb75f9040 2024-12-04T14:57:25,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/7709c9868b1a4403bd9c224fe570b9a0 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/7709c9868b1a4403bd9c224fe570b9a0 2024-12-04T14:57:25,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/aeea0405c311411d9a21229be343cbcc to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/aeea0405c311411d9a21229be343cbcc 2024-12-04T14:57:25,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b7337ba92b8149a48333225973bc6e60 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/b7337ba92b8149a48333225973bc6e60 2024-12-04T14:57:25,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af6093df62af4e13892c19117330a6bd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/af6093df62af4e13892c19117330a6bd 2024-12-04T14:57:25,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fcf7f24835c6450d894ffb2ea3aa0e98, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/65b15febf8e34276bd34a27b08bda93a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6d418f6c98874ec0b6836ecaac0fd265, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ad73d7bc79f04670b1a065713ee8a643, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0227db87db44456d929bd3fb47229165, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/c51517a4a3b7485aad6207483db597ff, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0e4196e6b8f047f59fc034cb94a37cfd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6747c025fac5428cb7563c26b66d9c5c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2e7efddb0d8d49bdbf42eecbd00a6a28, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dc0a2bef39ca4487864b5377974a71d4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fe215c2197d64b18acdf6e7a8dbc4b1d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/7c02c161e3e94a7288490bd83dc23662, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2817593ff8a14023ae0a3f424dedf93e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fc605d2aeac84cd2a846d94a17f471e6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/98e6f755f0a34d4d86857b14b7d96115, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a3a6b16cdf9f4f9391fafc5c71efc03b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/bbf6ef22658f43e29e218a3f9eba1bd7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/31226912a61c43bc92f30ca1a612ded4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/39f323e231d54072a2a3ab7b198d26bc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a970aa0fe5774e22acd2218d6e20b9e2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/5097c786e3904c32a17035a9df8b31f7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ee3e22f8194747f1b4b1005ed95a9321, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/d44ef1b2c9924ba29e0050b443bcacfd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/45b8a716ecb74ca5a955c46784c5c5e9, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dfd69ba8f2cf4133a0a2663748503688, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/cd31ce750c6e4fa5ad95c5c43749de79, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/36c031e268564ce3ae70a7c47aa9a842, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0055091b35e14877ba3081b42af5d851, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/26f05b241e574e48ab2a7169c7327c68] to archive 2024-12-04T14:57:25,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:57:25,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fcf7f24835c6450d894ffb2ea3aa0e98 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fcf7f24835c6450d894ffb2ea3aa0e98 2024-12-04T14:57:25,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/65b15febf8e34276bd34a27b08bda93a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/65b15febf8e34276bd34a27b08bda93a 2024-12-04T14:57:25,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6d418f6c98874ec0b6836ecaac0fd265 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6d418f6c98874ec0b6836ecaac0fd265 2024-12-04T14:57:25,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ad73d7bc79f04670b1a065713ee8a643 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ad73d7bc79f04670b1a065713ee8a643 2024-12-04T14:57:25,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0227db87db44456d929bd3fb47229165 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0227db87db44456d929bd3fb47229165 2024-12-04T14:57:25,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/c51517a4a3b7485aad6207483db597ff to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/c51517a4a3b7485aad6207483db597ff 2024-12-04T14:57:25,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0e4196e6b8f047f59fc034cb94a37cfd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0e4196e6b8f047f59fc034cb94a37cfd 2024-12-04T14:57:25,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6747c025fac5428cb7563c26b66d9c5c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/6747c025fac5428cb7563c26b66d9c5c 2024-12-04T14:57:25,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2e7efddb0d8d49bdbf42eecbd00a6a28 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2e7efddb0d8d49bdbf42eecbd00a6a28 2024-12-04T14:57:25,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dc0a2bef39ca4487864b5377974a71d4 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dc0a2bef39ca4487864b5377974a71d4 2024-12-04T14:57:25,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fe215c2197d64b18acdf6e7a8dbc4b1d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fe215c2197d64b18acdf6e7a8dbc4b1d 2024-12-04T14:57:25,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/7c02c161e3e94a7288490bd83dc23662 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/7c02c161e3e94a7288490bd83dc23662 2024-12-04T14:57:25,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2817593ff8a14023ae0a3f424dedf93e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/2817593ff8a14023ae0a3f424dedf93e 2024-12-04T14:57:25,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fc605d2aeac84cd2a846d94a17f471e6 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/fc605d2aeac84cd2a846d94a17f471e6 2024-12-04T14:57:25,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/98e6f755f0a34d4d86857b14b7d96115 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/98e6f755f0a34d4d86857b14b7d96115 2024-12-04T14:57:25,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a3a6b16cdf9f4f9391fafc5c71efc03b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a3a6b16cdf9f4f9391fafc5c71efc03b 2024-12-04T14:57:25,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/bbf6ef22658f43e29e218a3f9eba1bd7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/bbf6ef22658f43e29e218a3f9eba1bd7 2024-12-04T14:57:25,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/31226912a61c43bc92f30ca1a612ded4 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/31226912a61c43bc92f30ca1a612ded4 2024-12-04T14:57:25,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/39f323e231d54072a2a3ab7b198d26bc to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/39f323e231d54072a2a3ab7b198d26bc 2024-12-04T14:57:25,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a970aa0fe5774e22acd2218d6e20b9e2 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/a970aa0fe5774e22acd2218d6e20b9e2 2024-12-04T14:57:25,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/5097c786e3904c32a17035a9df8b31f7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/5097c786e3904c32a17035a9df8b31f7 2024-12-04T14:57:25,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ee3e22f8194747f1b4b1005ed95a9321 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ee3e22f8194747f1b4b1005ed95a9321 2024-12-04T14:57:25,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/d44ef1b2c9924ba29e0050b443bcacfd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/d44ef1b2c9924ba29e0050b443bcacfd 2024-12-04T14:57:25,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/45b8a716ecb74ca5a955c46784c5c5e9 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/45b8a716ecb74ca5a955c46784c5c5e9 2024-12-04T14:57:25,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dfd69ba8f2cf4133a0a2663748503688 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/dfd69ba8f2cf4133a0a2663748503688 2024-12-04T14:57:25,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/cd31ce750c6e4fa5ad95c5c43749de79 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/cd31ce750c6e4fa5ad95c5c43749de79 2024-12-04T14:57:25,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/36c031e268564ce3ae70a7c47aa9a842 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/36c031e268564ce3ae70a7c47aa9a842 2024-12-04T14:57:25,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0055091b35e14877ba3081b42af5d851 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/0055091b35e14877ba3081b42af5d851 2024-12-04T14:57:25,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/26f05b241e574e48ab2a7169c7327c68 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/26f05b241e574e48ab2a7169c7327c68 2024-12-04T14:57:25,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/11cb0c75b4ec4ea0955bae099a205062, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ddc5b69220340fbb1371420bdc93b5e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/e59bba0b45de4ae49804a23428cde443, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f6f7dc317f144475908ff1845b815727, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/a84f8c7e4efd46e785c3bac64a31baf8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4e2f45b81916481c914993385a0dd04a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/34841bd6a4544788a9c4e8d6700abc84, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/3475034df6bb4064b368f30ea4c9fa17, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/6a58041c4cab40da9dd04e6e5c39f4de, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/85cabd4ee2884d9f982c1525d1874d33, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/408dc63aa6f84c138cbae96b083dbd2f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/d97486b39a3045b1b0f24c548b89d6f7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/2edc3fcbe9644af4bde97c6dea715629, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/08316001014e47d98ea9d3e4cf290eab, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/c924ae000ddb4182ad218db19915b276, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4bd98c3d7a12402e9c2f289b412c1966, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/53b3fc0b485249daad3c73898fde6354, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/bf5447f9e41843699a6e4aabcb964839, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/7244de35be864e4585a05b63db5721f4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/61dc8ae25c7b453bb28818636447e08c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/298c403940304eb394a27053fc54882d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/16e658ad308e4cfe93d1a52ffcdabc37, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/8d49502565594954ab2e817f34d180d9, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/fb977cbf2a65453cace4349df9871329, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/1418d2fb969d418d90ebc919db44ed95, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ecdcca69e804a47a3784e926951bb70, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/b76352754bf94b84a8e176aaaea41c50, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f08f4f1c38d949239ea6cdf8e41fb026, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/911e7c801d49453fa41db21d579993ad] to archive 2024-12-04T14:57:25,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:57:25,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/11cb0c75b4ec4ea0955bae099a205062 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/11cb0c75b4ec4ea0955bae099a205062 2024-12-04T14:57:25,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ddc5b69220340fbb1371420bdc93b5e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ddc5b69220340fbb1371420bdc93b5e 2024-12-04T14:57:25,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/e59bba0b45de4ae49804a23428cde443 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/e59bba0b45de4ae49804a23428cde443 2024-12-04T14:57:25,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f6f7dc317f144475908ff1845b815727 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f6f7dc317f144475908ff1845b815727 2024-12-04T14:57:25,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/a84f8c7e4efd46e785c3bac64a31baf8 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/a84f8c7e4efd46e785c3bac64a31baf8 2024-12-04T14:57:25,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4e2f45b81916481c914993385a0dd04a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4e2f45b81916481c914993385a0dd04a 2024-12-04T14:57:25,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/34841bd6a4544788a9c4e8d6700abc84 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/34841bd6a4544788a9c4e8d6700abc84 2024-12-04T14:57:25,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/3475034df6bb4064b368f30ea4c9fa17 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/3475034df6bb4064b368f30ea4c9fa17 2024-12-04T14:57:25,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/6a58041c4cab40da9dd04e6e5c39f4de to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/6a58041c4cab40da9dd04e6e5c39f4de 2024-12-04T14:57:25,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/85cabd4ee2884d9f982c1525d1874d33 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/85cabd4ee2884d9f982c1525d1874d33 2024-12-04T14:57:25,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/408dc63aa6f84c138cbae96b083dbd2f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/408dc63aa6f84c138cbae96b083dbd2f 2024-12-04T14:57:25,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/d97486b39a3045b1b0f24c548b89d6f7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/d97486b39a3045b1b0f24c548b89d6f7 2024-12-04T14:57:25,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/2edc3fcbe9644af4bde97c6dea715629 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/2edc3fcbe9644af4bde97c6dea715629 2024-12-04T14:57:25,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/08316001014e47d98ea9d3e4cf290eab to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/08316001014e47d98ea9d3e4cf290eab 2024-12-04T14:57:25,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/c924ae000ddb4182ad218db19915b276 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/c924ae000ddb4182ad218db19915b276 2024-12-04T14:57:25,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4bd98c3d7a12402e9c2f289b412c1966 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/4bd98c3d7a12402e9c2f289b412c1966 2024-12-04T14:57:25,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/53b3fc0b485249daad3c73898fde6354 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/53b3fc0b485249daad3c73898fde6354 2024-12-04T14:57:25,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/bf5447f9e41843699a6e4aabcb964839 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/bf5447f9e41843699a6e4aabcb964839 2024-12-04T14:57:25,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/7244de35be864e4585a05b63db5721f4 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/7244de35be864e4585a05b63db5721f4 2024-12-04T14:57:25,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/61dc8ae25c7b453bb28818636447e08c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/61dc8ae25c7b453bb28818636447e08c 2024-12-04T14:57:25,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/298c403940304eb394a27053fc54882d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/298c403940304eb394a27053fc54882d 2024-12-04T14:57:25,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/16e658ad308e4cfe93d1a52ffcdabc37 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/16e658ad308e4cfe93d1a52ffcdabc37 2024-12-04T14:57:25,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/8d49502565594954ab2e817f34d180d9 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/8d49502565594954ab2e817f34d180d9 2024-12-04T14:57:25,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/fb977cbf2a65453cace4349df9871329 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/fb977cbf2a65453cace4349df9871329 2024-12-04T14:57:25,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/1418d2fb969d418d90ebc919db44ed95 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/1418d2fb969d418d90ebc919db44ed95 2024-12-04T14:57:25,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ecdcca69e804a47a3784e926951bb70 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/5ecdcca69e804a47a3784e926951bb70 2024-12-04T14:57:25,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/b76352754bf94b84a8e176aaaea41c50 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/b76352754bf94b84a8e176aaaea41c50 2024-12-04T14:57:25,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f08f4f1c38d949239ea6cdf8e41fb026 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/f08f4f1c38d949239ea6cdf8e41fb026 2024-12-04T14:57:25,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/911e7c801d49453fa41db21d579993ad to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/911e7c801d49453fa41db21d579993ad 2024-12-04T14:57:25,488 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/recovered.edits/432.seqid, newMaxSeqId=432, maxSeqId=1 2024-12-04T14:57:25,488 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. 2024-12-04T14:57:25,488 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 0fa368e0aeee714f048ec4ca8c747b4d: Waiting for close lock at 1733324244017Running coprocessor pre-close hooks at 1733324244017Disabling compacts and flushes for region at 1733324244017Disabling writes for close at 1733324244017Obtaining lock to block concurrent updates at 1733324244018 (+1 ms)Preparing flush snapshotting stores in 0fa368e0aeee714f048ec4ca8c747b4d at 1733324244018Finished memstore snapshotting TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1733324244018Flushing stores of TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d. at 1733324244020 (+2 ms)Flushing 0fa368e0aeee714f048ec4ca8c747b4d/A: creating writer at 1733324244020Flushing 0fa368e0aeee714f048ec4ca8c747b4d/A: appending metadata at 1733324244024 (+4 ms)Flushing 0fa368e0aeee714f048ec4ca8c747b4d/A: closing flushed file at 1733324244024Flushing 0fa368e0aeee714f048ec4ca8c747b4d/B: creating writer at 1733324244439 (+415 ms)Flushing 0fa368e0aeee714f048ec4ca8c747b4d/B: appending metadata at 1733324244441 (+2 ms)Flushing 0fa368e0aeee714f048ec4ca8c747b4d/B: closing flushed file at 1733324244441Flushing 0fa368e0aeee714f048ec4ca8c747b4d/C: creating writer at 1733324244858 (+417 ms)Flushing 0fa368e0aeee714f048ec4ca8c747b4d/C: appending metadata at 1733324244860 (+2 ms)Flushing 0fa368e0aeee714f048ec4ca8c747b4d/C: closing flushed file at 1733324244860Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a2b59a1: reopening flushed file at 1733324245389 (+529 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fa99e12: reopening flushed file at 1733324245396 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e88f8ea: reopening flushed file at 1733324245403 (+7 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 0fa368e0aeee714f048ec4ca8c747b4d in 1389ms, sequenceid=429, compaction requested=false at 1733324245407 (+4 ms)Writing region close event to WAL at 1733324245485 (+78 ms)Running coprocessor post-close hooks at 1733324245488 (+3 ms)Closed at 1733324245488 2024-12-04T14:57:25,490 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:25,491 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=0fa368e0aeee714f048ec4ca8c747b4d, regionState=CLOSED 2024-12-04T14:57:25,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:57:25,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-04T14:57:25,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 0fa368e0aeee714f048ec4ca8c747b4d, server=d214edae7bd2,34477,1733324158096 in 1.6300 sec 2024-12-04T14:57:25,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-04T14:57:25,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fa368e0aeee714f048ec4ca8c747b4d, UNASSIGN in 1.6340 sec 2024-12-04T14:57:25,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-04T14:57:25,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6370 sec 2024-12-04T14:57:25,498 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324245497"}]},"ts":"1733324245497"} 2024-12-04T14:57:25,499 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T14:57:25,499 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T14:57:25,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6480 sec 2024-12-04T14:57:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T14:57:26,006 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:26,007 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T14:57:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:26,010 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-04T14:57:26,011 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:26,015 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:26,018 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/recovered.edits] 2024-12-04T14:57:26,021 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1f81d16a481c456ca1a1263fc532b611 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/1f81d16a481c456ca1a1263fc532b611 2024-12-04T14:57:26,023 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/499a048387234162aa98f3205ce276ea to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/A/499a048387234162aa98f3205ce276ea 2024-12-04T14:57:26,027 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/bd206f5828da4ec8a85ecc5727b7fa3a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/bd206f5828da4ec8a85ecc5727b7fa3a 2024-12-04T14:57:26,028 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ecb4c7a2c1474d4bb51cfb9c5a4a030b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/B/ecb4c7a2c1474d4bb51cfb9c5a4a030b 2024-12-04T14:57:26,032 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/8cdf9873c0f24c878902b080234afd4b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/8cdf9873c0f24c878902b080234afd4b 2024-12-04T14:57:26,033 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/d587147640724d4c8491c5c73aeab32f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/C/d587147640724d4c8491c5c73aeab32f 2024-12-04T14:57:26,037 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/recovered.edits/432.seqid to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d/recovered.edits/432.seqid 2024-12-04T14:57:26,038 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0fa368e0aeee714f048ec4ca8c747b4d 2024-12-04T14:57:26,038 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T14:57:26,041 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:26,045 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T14:57:26,047 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T14:57:26,048 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:26,048 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T14:57:26,048 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324246048"}]},"ts":"9223372036854775807"} 2024-12-04T14:57:26,050 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T14:57:26,050 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0fa368e0aeee714f048ec4ca8c747b4d, NAME => 'TestAcidGuarantees,,1733324216472.0fa368e0aeee714f048ec4ca8c747b4d.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T14:57:26,050 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T14:57:26,050 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324246050"}]},"ts":"9223372036854775807"} 2024-12-04T14:57:26,052 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T14:57:26,053 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:26,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-12-04T14:57:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-04T14:57:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T14:57:26,120 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:26,131 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 245), OpenFileDescriptor=455 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=348 (was 352), ProcessCount=11 (was 11), AvailableMemoryMB=8713 (was 8805) 2024-12-04T14:57:26,139 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=348, ProcessCount=11, AvailableMemoryMB=8713 2024-12-04T14:57:26,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:57:26,140 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:57:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:26,143 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:57:26,143 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:26,144 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-12-04T14:57:26,144 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:57:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T14:57:26,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742127_1303 (size=959) 2024-12-04T14:57:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T14:57:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T14:57:26,556 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1 2024-12-04T14:57:26,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742128_1304 (size=53) 2024-12-04T14:57:26,577 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:57:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T14:57:26,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:57:26,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0ed254c309e31d6514a1b829d83e0676, disabling compactions & flushes 2024-12-04T14:57:26,969 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:26,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:26,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. after waiting 0 ms 2024-12-04T14:57:26,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:26,969 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:26,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ed254c309e31d6514a1b829d83e0676: Waiting for close lock at 1733324246969Disabling compacts and flushes for region at 1733324246969Disabling writes for close at 1733324246969Writing region close event to WAL at 1733324246969Closed at 1733324246969 2024-12-04T14:57:26,972 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:57:26,973 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324246972"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324246972"}]},"ts":"1733324246972"} 2024-12-04T14:57:26,976 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T14:57:26,977 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:57:26,977 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324246977"}]},"ts":"1733324246977"} 2024-12-04T14:57:26,979 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T14:57:26,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, ASSIGN}] 2024-12-04T14:57:26,980 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, ASSIGN 2024-12-04T14:57:26,982 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, ASSIGN; state=OFFLINE, location=d214edae7bd2,34477,1733324158096; forceNewPlan=false, retain=false 2024-12-04T14:57:27,133 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=0ed254c309e31d6514a1b829d83e0676, regionState=OPENING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:27,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, ASSIGN because future has completed 2024-12-04T14:57:27,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:57:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T14:57:27,296 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:27,297 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:57:27,297 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,297 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:57:27,298 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,298 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,300 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,302 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ed254c309e31d6514a1b829d83e0676 columnFamilyName A 2024-12-04T14:57:27,302 DEBUG [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:27,302 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(327): Store=0ed254c309e31d6514a1b829d83e0676/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:57:27,303 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,304 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ed254c309e31d6514a1b829d83e0676 columnFamilyName B 2024-12-04T14:57:27,304 DEBUG [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:27,305 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(327): Store=0ed254c309e31d6514a1b829d83e0676/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:57:27,305 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,307 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ed254c309e31d6514a1b829d83e0676 columnFamilyName C 2024-12-04T14:57:27,307 DEBUG [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:27,307 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(327): Store=0ed254c309e31d6514a1b829d83e0676/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:57:27,308 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,308 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,309 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,310 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,310 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,311 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:57:27,313 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,315 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:57:27,316 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 0ed254c309e31d6514a1b829d83e0676; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60167330, jitterRate=-0.1034369170665741}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:57:27,316 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:27,316 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 0ed254c309e31d6514a1b829d83e0676: Running coprocessor pre-open hook at 1733324247298Writing region info on filesystem at 1733324247298Initializing all the Stores at 1733324247299 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324247299Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324247300 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324247300Cleaning up temporary data from old regions at 1733324247311 (+11 ms)Running coprocessor post-open hooks at 1733324247316 (+5 ms)Region opened successfully at 1733324247316 2024-12-04T14:57:27,317 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., pid=87, masterSystemTime=1733324247290 2024-12-04T14:57:27,318 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:27,318 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:27,319 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=0ed254c309e31d6514a1b829d83e0676, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:27,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:57:27,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-04T14:57:27,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 in 184 msec 2024-12-04T14:57:27,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-04T14:57:27,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, ASSIGN in 343 msec 2024-12-04T14:57:27,324 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:57:27,324 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324247324"}]},"ts":"1733324247324"} 2024-12-04T14:57:27,325 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T14:57:27,325 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:57:27,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-12-04T14:57:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T14:57:28,296 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:28,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:28,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:28,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:28,301 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:28,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:28,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:28,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636a19e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:28,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:28,305 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@954f35e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:28,306 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:28,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:28,308 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:57:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,45797,1733324157426 2024-12-04T14:57:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@441e78e1 2024-12-04T14:57:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:57:28,310 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:57:28,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:57:28,311 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:57:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:28,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742129_1305 (size=995) 2024-12-04T14:57:28,727 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-04T14:57:28,727 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-04T14:57:28,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:57:28,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, REOPEN/MOVE}] 2024-12-04T14:57:28,738 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, REOPEN/MOVE 2024-12-04T14:57:28,739 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=0ed254c309e31d6514a1b829d83e0676, regionState=CLOSING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:28,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, REOPEN/MOVE because future has completed 2024-12-04T14:57:28,741 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:57:28,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:57:28,895 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:28,895 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:57:28,895 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 0ed254c309e31d6514a1b829d83e0676, disabling compactions & flushes 2024-12-04T14:57:28,895 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:28,895 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:28,895 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. after waiting 0 ms 2024-12-04T14:57:28,895 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:28,900 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T14:57:28,901 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:28,901 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 0ed254c309e31d6514a1b829d83e0676: Waiting for close lock at 1733324248895Running coprocessor pre-close hooks at 1733324248895Disabling compacts and flushes for region at 1733324248895Disabling writes for close at 1733324248895Writing region close event to WAL at 1733324248896 (+1 ms)Running coprocessor post-close hooks at 1733324248900 (+4 ms)Closed at 1733324248901 (+1 ms) 2024-12-04T14:57:28,901 WARN [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 0ed254c309e31d6514a1b829d83e0676 to self. 2024-12-04T14:57:28,903 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:28,904 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=0ed254c309e31d6514a1b829d83e0676, regionState=CLOSED 2024-12-04T14:57:28,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:57:28,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-04T14:57:28,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 in 166 msec 2024-12-04T14:57:28,910 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, REOPEN/MOVE; state=CLOSED, location=d214edae7bd2,34477,1733324158096; forceNewPlan=false, retain=true 2024-12-04T14:57:29,061 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=0ed254c309e31d6514a1b829d83e0676, regionState=OPENING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, REOPEN/MOVE because future has completed 2024-12-04T14:57:29,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:57:29,222 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,222 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:57:29,223 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,223 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:57:29,223 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,223 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,225 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,227 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ed254c309e31d6514a1b829d83e0676 columnFamilyName A 2024-12-04T14:57:29,229 DEBUG [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:29,230 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(327): Store=0ed254c309e31d6514a1b829d83e0676/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:57:29,231 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,232 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ed254c309e31d6514a1b829d83e0676 columnFamilyName B 2024-12-04T14:57:29,232 DEBUG [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:29,233 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(327): Store=0ed254c309e31d6514a1b829d83e0676/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:57:29,233 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,234 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ed254c309e31d6514a1b829d83e0676 columnFamilyName C 2024-12-04T14:57:29,234 DEBUG [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:29,234 INFO [StoreOpener-0ed254c309e31d6514a1b829d83e0676-1 {}] regionserver.HStore(327): Store=0ed254c309e31d6514a1b829d83e0676/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:57:29,234 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,235 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,236 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,237 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,237 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,238 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:57:29,239 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,240 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 0ed254c309e31d6514a1b829d83e0676; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59203039, jitterRate=-0.11780597269535065}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:57:29,240 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,241 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 0ed254c309e31d6514a1b829d83e0676: Running coprocessor pre-open hook at 1733324249223Writing region info on filesystem at 1733324249223Initializing all the Stores at 1733324249225 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324249225Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324249225Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324249225Cleaning up temporary data from old regions at 1733324249237 (+12 ms)Running coprocessor post-open hooks at 1733324249240 (+3 ms)Region opened successfully at 1733324249241 (+1 ms) 2024-12-04T14:57:29,242 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., pid=92, masterSystemTime=1733324249217 2024-12-04T14:57:29,244 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,244 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=0ed254c309e31d6514a1b829d83e0676, regionState=OPEN, openSeqNum=5, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:57:29,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-12-04T14:57:29,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 in 183 msec 2024-12-04T14:57:29,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-04T14:57:29,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, REOPEN/MOVE in 514 msec 2024-12-04T14:57:29,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-04T14:57:29,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-12-04T14:57:29,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 944 msec 2024-12-04T14:57:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-04T14:57:29,260 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:29,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:29,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:29,262 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e15df87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:29,265 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570c3fc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:29,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:29,268 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23522aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:29,270 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b22b15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:29,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:29,273 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b9a267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:29,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:29,276 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72a02f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,278 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:29,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:29,279 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c34b6b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:29,280 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57932262, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:29,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:29,281 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e9f6f7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:29,283 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ffba2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:29,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:29,284 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@857a081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:29,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:29,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:29,285 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443697b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:29,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:29,287 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44648d5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:29,288 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:29,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f7a85a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:29,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:29,290 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586098f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:29,291 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:29,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf69aef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:29,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:29,293 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1977f314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:29,294 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:29,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f0c510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:29,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:29,297 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df9d31d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:29,298 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23fe48f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:29,300 DEBUG [Thread-1419 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:29,301 DEBUG [Thread-1421 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:29,301 DEBUG [Thread-1408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:29,301 DEBUG [Thread-1417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:29,301 DEBUG [Thread-1425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:29,302 DEBUG [Thread-1412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:29,302 DEBUG [Thread-1414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:29,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:29,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:29,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:29,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:29,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:29,304 DEBUG [Thread-1406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:29,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:57:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:29,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:29,305 DEBUG [Thread-1410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:29,305 DEBUG [Thread-1423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-04T14:57:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:29,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:57:29,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:29,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:29,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:29,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:57:29,306 DEBUG [Thread-1423 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:29,306 DEBUG [Thread-1423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T14:57:29,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:29,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:29,307 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:57:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:57:29,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:29,308 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:29,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:57:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:57:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:57:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:57:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:57:29,318 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:29,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:57:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e19d2365109d4d4f8c784580474ece34_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324249318/Put/seqid=0 2024-12-04T14:57:29,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742130_1306 (size=14594) 2024-12-04T14:57:29,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324309348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324309348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324309349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324309349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324309349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T14:57:29,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324309455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324309465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324309465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324309465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324309465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:29,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:29,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T14:57:29,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324309665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324309675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324309675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324309675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324309675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,744 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:29,747 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e19d2365109d4d4f8c784580474ece34_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e19d2365109d4d4f8c784580474ece34_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:29,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/a9d32e00b46042909cbf9816eb948576, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:29,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/a9d32e00b46042909cbf9816eb948576 is 175, key is test_row_0/A:col10/1733324249318/Put/seqid=0 2024-12-04T14:57:29,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742131_1307 (size=39549) 2024-12-04T14:57:29,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:29,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:29,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:29,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T14:57:29,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324309974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324309985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324309985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324309985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:29,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324309985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:30,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,153 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/a9d32e00b46042909cbf9816eb948576 2024-12-04T14:57:30,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/b00f182635b140e4a9bf394f77a769b4 is 50, key is test_row_0/B:col10/1733324249318/Put/seqid=0 2024-12-04T14:57:30,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742132_1308 (size=12001) 2024-12-04T14:57:30,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:30,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:30,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T14:57:30,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324310484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:30,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324310495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:30,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324310495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:30,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324310495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:30,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324310495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:30,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/b00f182635b140e4a9bf394f77a769b4 2024-12-04T14:57:30,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/0d27dccf06b2488fb7c462183b9b45fa is 50, key is test_row_0/C:col10/1733324249318/Put/seqid=0 2024-12-04T14:57:30,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742133_1309 (size=12001) 2024-12-04T14:57:30,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:30,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:30,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:30,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:30,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:30,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:31,015 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:57:31,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/0d27dccf06b2488fb7c462183b9b45fa 2024-12-04T14:57:31,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/a9d32e00b46042909cbf9816eb948576 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a9d32e00b46042909cbf9816eb948576 2024-12-04T14:57:31,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a9d32e00b46042909cbf9816eb948576, entries=200, sequenceid=16, filesize=38.6 K 2024-12-04T14:57:31,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/b00f182635b140e4a9bf394f77a769b4 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/b00f182635b140e4a9bf394f77a769b4 2024-12-04T14:57:31,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/b00f182635b140e4a9bf394f77a769b4, entries=150, sequenceid=16, filesize=11.7 K 2024-12-04T14:57:31,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/0d27dccf06b2488fb7c462183b9b45fa as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0d27dccf06b2488fb7c462183b9b45fa 2024-12-04T14:57:31,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0d27dccf06b2488fb7c462183b9b45fa, entries=150, sequenceid=16, filesize=11.7 K 2024-12-04T14:57:31,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0ed254c309e31d6514a1b829d83e0676 in 1721ms, sequenceid=16, compaction requested=false 2024-12-04T14:57:31,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:31,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T14:57:31,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:31,145 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-04T14:57:31,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a204242c34524abdba26320bcd07b2fe_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324249348/Put/seqid=0 2024-12-04T14:57:31,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742134_1310 (size=12154) 2024-12-04T14:57:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T14:57:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:31,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:31,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324311515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324311515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,519 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:31,519 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_2' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324311515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324311516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324311516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:31,562 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a204242c34524abdba26320bcd07b2fe_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a204242c34524abdba26320bcd07b2fe_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:31,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/7c6f26e6d723417b9252a9cdd762ca36, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:31,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/7c6f26e6d723417b9252a9cdd762ca36 is 175, key is test_row_0/A:col10/1733324249348/Put/seqid=0 2024-12-04T14:57:31,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742135_1311 (size=30955) 2024-12-04T14:57:31,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324311625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324311625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324311625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324311835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324311835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324311835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:31,968 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/7c6f26e6d723417b9252a9cdd762ca36 2024-12-04T14:57:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/9f6d4231663b437b916f8faabdc4d509 is 50, key is test_row_0/B:col10/1733324249348/Put/seqid=0 2024-12-04T14:57:31,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742136_1312 (size=12001) 2024-12-04T14:57:32,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324312145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:32,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324312145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:32,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324312145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:32,380 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/9f6d4231663b437b916f8faabdc4d509 2024-12-04T14:57:32,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/7990148f04694410a6ac868ce03dc9bc is 50, key is test_row_0/C:col10/1733324249348/Put/seqid=0 2024-12-04T14:57:32,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742137_1313 (size=12001) 2024-12-04T14:57:32,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324312655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:32,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324312655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:32,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324312655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:32,789 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/7990148f04694410a6ac868ce03dc9bc 2024-12-04T14:57:32,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/7c6f26e6d723417b9252a9cdd762ca36 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7c6f26e6d723417b9252a9cdd762ca36 2024-12-04T14:57:32,796 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7c6f26e6d723417b9252a9cdd762ca36, entries=150, sequenceid=40, filesize=30.2 K 2024-12-04T14:57:32,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/9f6d4231663b437b916f8faabdc4d509 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/9f6d4231663b437b916f8faabdc4d509 2024-12-04T14:57:32,801 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/9f6d4231663b437b916f8faabdc4d509, entries=150, sequenceid=40, filesize=11.7 K 2024-12-04T14:57:32,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/7990148f04694410a6ac868ce03dc9bc as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/7990148f04694410a6ac868ce03dc9bc 2024-12-04T14:57:32,805 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/7990148f04694410a6ac868ce03dc9bc, entries=150, sequenceid=40, filesize=11.7 K 2024-12-04T14:57:32,806 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 0ed254c309e31d6514a1b829d83e0676 in 1661ms, sequenceid=40, compaction requested=false 2024-12-04T14:57:32,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:32,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:32,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-04T14:57:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-04T14:57:32,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-04T14:57:32,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4990 sec 2024-12-04T14:57:32,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.5050 sec 2024-12-04T14:57:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T14:57:33,476 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:33,476 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-04T14:57:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T14:57:33,478 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:33,478 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:33,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:33,539 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:33,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-04T14:57:33,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042a0a442d0fee4837a80be4f1d04afee3_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324253538/Put/seqid=0 2024-12-04T14:57:33,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742138_1314 (size=12154) 2024-12-04T14:57:33,548 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:33,553 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042a0a442d0fee4837a80be4f1d04afee3_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042a0a442d0fee4837a80be4f1d04afee3_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:33,553 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/c33310f6384242d4886a21a6ab7899a1, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:33,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/c33310f6384242d4886a21a6ab7899a1 is 175, key is test_row_0/A:col10/1733324253538/Put/seqid=0 2024-12-04T14:57:33,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742139_1315 (size=30955) 2024-12-04T14:57:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T14:57:33,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324313606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:33,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324313610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:33,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T14:57:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:33,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:33,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324313675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:33,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324313675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:33,678 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:33,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324313675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:33,678 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:33,678 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:33,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324313725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:33,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324313725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:33,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T14:57:33,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:33,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:33,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:33,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:33,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T14:57:33,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T14:57:33,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:33,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:33,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:33,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:33,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:33,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324313935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:33,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:33,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324313935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:33,958 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/c33310f6384242d4886a21a6ab7899a1 2024-12-04T14:57:33,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/21a9f0cc3cd24921933b025a979bf6a5 is 50, key is test_row_0/B:col10/1733324253538/Put/seqid=0 2024-12-04T14:57:33,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742140_1316 (size=12001) 2024-12-04T14:57:34,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T14:57:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T14:57:34,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T14:57:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324314254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:34,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:34,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324314254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:34,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/21a9f0cc3cd24921933b025a979bf6a5 2024-12-04T14:57:34,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/fd9aaffc8c9a4eaea9c8de5fe4999d1a is 50, key is test_row_0/C:col10/1733324253538/Put/seqid=0 2024-12-04T14:57:34,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742141_1317 (size=12001) 2024-12-04T14:57:34,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T14:57:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T14:57:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T14:57:34,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T14:57:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:34,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324314765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:34,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324314765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:34,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/fd9aaffc8c9a4eaea9c8de5fe4999d1a 2024-12-04T14:57:34,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/c33310f6384242d4886a21a6ab7899a1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/c33310f6384242d4886a21a6ab7899a1 2024-12-04T14:57:34,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/c33310f6384242d4886a21a6ab7899a1, entries=150, sequenceid=54, filesize=30.2 K 2024-12-04T14:57:34,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/21a9f0cc3cd24921933b025a979bf6a5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/21a9f0cc3cd24921933b025a979bf6a5 2024-12-04T14:57:34,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/21a9f0cc3cd24921933b025a979bf6a5, entries=150, sequenceid=54, filesize=11.7 K 2024-12-04T14:57:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/fd9aaffc8c9a4eaea9c8de5fe4999d1a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/fd9aaffc8c9a4eaea9c8de5fe4999d1a 2024-12-04T14:57:34,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/fd9aaffc8c9a4eaea9c8de5fe4999d1a, entries=150, sequenceid=54, filesize=11.7 K 2024-12-04T14:57:34,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 0ed254c309e31d6514a1b829d83e0676 in 1256ms, sequenceid=54, compaction requested=true 2024-12-04T14:57:34,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:34,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:34,795 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:34,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:34,796 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:34,797 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:34,797 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:34,797 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/B is initiating minor compaction (all files) 2024-12-04T14:57:34,797 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/A is initiating minor compaction (all files) 2024-12-04T14:57:34,797 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/B in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,797 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/b00f182635b140e4a9bf394f77a769b4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/9f6d4231663b437b916f8faabdc4d509, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/21a9f0cc3cd24921933b025a979bf6a5] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=35.2 K 2024-12-04T14:57:34,797 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/A in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,797 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a9d32e00b46042909cbf9816eb948576, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7c6f26e6d723417b9252a9cdd762ca36, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/c33310f6384242d4886a21a6ab7899a1] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=99.1 K 2024-12-04T14:57:34,797 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,797 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a9d32e00b46042909cbf9816eb948576, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7c6f26e6d723417b9252a9cdd762ca36, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/c33310f6384242d4886a21a6ab7899a1] 2024-12-04T14:57:34,797 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b00f182635b140e4a9bf394f77a769b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324249314 2024-12-04T14:57:34,797 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d32e00b46042909cbf9816eb948576, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324249314 2024-12-04T14:57:34,798 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f6d4231663b437b916f8faabdc4d509, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733324249347 2024-12-04T14:57:34,798 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6f26e6d723417b9252a9cdd762ca36, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733324249347 2024-12-04T14:57:34,798 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a9f0cc3cd24921933b025a979bf6a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733324251508 2024-12-04T14:57:34,798 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting c33310f6384242d4886a21a6ab7899a1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733324251508 2024-12-04T14:57:34,803 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:34,805 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#B#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:34,805 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/fe024119a6fb4063931b83b9d3c74a2b is 50, key is test_row_0/B:col10/1733324253538/Put/seqid=0 2024-12-04T14:57:34,811 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204f642b20617ee4da0b2fb277f8fe49219_0ed254c309e31d6514a1b829d83e0676 store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:34,812 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204f642b20617ee4da0b2fb277f8fe49219_0ed254c309e31d6514a1b829d83e0676, store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:34,813 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f642b20617ee4da0b2fb277f8fe49219_0ed254c309e31d6514a1b829d83e0676 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:34,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742142_1318 (size=12104) 2024-12-04T14:57:34,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742143_1319 (size=4469) 2024-12-04T14:57:34,826 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#A#compaction#270 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:34,826 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/7f40cd6d67fe412c924f8c31a7dd3839 is 175, key is test_row_0/A:col10/1733324253538/Put/seqid=0 2024-12-04T14:57:34,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742144_1320 (size=31058) 2024-12-04T14:57:34,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T14:57:34,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:34,851 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-04T14:57:34,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204fec87a700e654d749260e775d4b558ee_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324253596/Put/seqid=0 2024-12-04T14:57:34,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742145_1321 (size=12154) 2024-12-04T14:57:35,219 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/fe024119a6fb4063931b83b9d3c74a2b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/fe024119a6fb4063931b83b9d3c74a2b 2024-12-04T14:57:35,231 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/B of 0ed254c309e31d6514a1b829d83e0676 into fe024119a6fb4063931b83b9d3c74a2b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:35,232 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:35,232 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/B, priority=13, startTime=1733324254796; duration=0sec 2024-12-04T14:57:35,232 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:35,232 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:B 2024-12-04T14:57:35,232 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:35,232 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:35,233 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/C is initiating minor compaction (all files) 2024-12-04T14:57:35,233 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/C in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:35,233 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0d27dccf06b2488fb7c462183b9b45fa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/7990148f04694410a6ac868ce03dc9bc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/fd9aaffc8c9a4eaea9c8de5fe4999d1a] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=35.2 K 2024-12-04T14:57:35,233 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d27dccf06b2488fb7c462183b9b45fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324249314 2024-12-04T14:57:35,233 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7990148f04694410a6ac868ce03dc9bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733324249347 2024-12-04T14:57:35,233 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting fd9aaffc8c9a4eaea9c8de5fe4999d1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733324251508 2024-12-04T14:57:35,238 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#C#compaction#273 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:35,239 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/0420f103d2c64cc688b2a4610b2f2e97 is 50, key is test_row_0/C:col10/1733324253538/Put/seqid=0 2024-12-04T14:57:35,245 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/7f40cd6d67fe412c924f8c31a7dd3839 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7f40cd6d67fe412c924f8c31a7dd3839 2024-12-04T14:57:35,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742146_1322 (size=12104) 2024-12-04T14:57:35,249 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/A of 0ed254c309e31d6514a1b829d83e0676 into 7f40cd6d67fe412c924f8c31a7dd3839(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:35,249 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:35,249 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/A, priority=13, startTime=1733324254795; duration=0sec 2024-12-04T14:57:35,249 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:35,249 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:A 2024-12-04T14:57:35,250 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/0420f103d2c64cc688b2a4610b2f2e97 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0420f103d2c64cc688b2a4610b2f2e97 2024-12-04T14:57:35,254 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/C of 0ed254c309e31d6514a1b829d83e0676 into 0420f103d2c64cc688b2a4610b2f2e97(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:35,254 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:35,254 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/C, priority=13, startTime=1733324254796; duration=0sec 2024-12-04T14:57:35,254 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:35,254 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:C 2024-12-04T14:57:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,264 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204fec87a700e654d749260e775d4b558ee_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204fec87a700e654d749260e775d4b558ee_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:35,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/fd6d4fbd4b754ebe96e1c946ca7b862c, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:35,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/fd6d4fbd4b754ebe96e1c946ca7b862c is 175, key is test_row_0/A:col10/1733324253596/Put/seqid=0 2024-12-04T14:57:35,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742147_1323 (size=30955) 2024-12-04T14:57:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T14:57:35,668 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/fd6d4fbd4b754ebe96e1c946ca7b862c 2024-12-04T14:57:35,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/6d22ff237e5a458bb96293c43019f26a is 50, key is test_row_0/B:col10/1733324253596/Put/seqid=0 2024-12-04T14:57:35,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742148_1324 (size=12001) 2024-12-04T14:57:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:35,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:35,691 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/6d22ff237e5a458bb96293c43019f26a 2024-12-04T14:57:35,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/c89e3a9f06a448c1967756132856124b is 50, key is test_row_0/C:col10/1733324253596/Put/seqid=0 2024-12-04T14:57:35,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742149_1325 (size=12001) 2024-12-04T14:57:35,706 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/c89e3a9f06a448c1967756132856124b 2024-12-04T14:57:35,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/fd6d4fbd4b754ebe96e1c946ca7b862c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/fd6d4fbd4b754ebe96e1c946ca7b862c 2024-12-04T14:57:35,715 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/fd6d4fbd4b754ebe96e1c946ca7b862c, entries=150, sequenceid=76, filesize=30.2 K 2024-12-04T14:57:35,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/6d22ff237e5a458bb96293c43019f26a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/6d22ff237e5a458bb96293c43019f26a 2024-12-04T14:57:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324315711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:35,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324315712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:35,721 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/6d22ff237e5a458bb96293c43019f26a, entries=150, sequenceid=76, filesize=11.7 K 2024-12-04T14:57:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/c89e3a9f06a448c1967756132856124b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/c89e3a9f06a448c1967756132856124b 2024-12-04T14:57:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,725 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/c89e3a9f06a448c1967756132856124b, entries=150, sequenceid=76, filesize=11.7 K 2024-12-04T14:57:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,726 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0ed254c309e31d6514a1b829d83e0676 in 875ms, sequenceid=76, compaction requested=false 2024-12-04T14:57:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-04T14:57:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-04T14:57:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-04T14:57:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2500 sec 2024-12-04T14:57:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.2550 sec 2024-12-04T14:57:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:35,749 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:35,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-04T14:57:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d5c312d115cb47c7a67c8ee137a91130_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324255710/Put/seqid=0 2024-12-04T14:57:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742150_1326 (size=12154) 2024-12-04T14:57:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:35,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324315824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324315824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324315825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324315824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:35,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324315826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:35,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324315945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324315945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324315945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324316044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324316044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324316154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324316154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324316155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,165 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:36,168 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d5c312d115cb47c7a67c8ee137a91130_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d5c312d115cb47c7a67c8ee137a91130_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:36,169 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/f567531131564d0dbfb2ec3463b31745, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:36,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/f567531131564d0dbfb2ec3463b31745 is 175, key is test_row_0/A:col10/1733324255710/Put/seqid=0 2024-12-04T14:57:36,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742151_1327 (size=30951) 2024-12-04T14:57:36,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324316355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324316355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324316465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324316465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324316465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,573 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/f567531131564d0dbfb2ec3463b31745 2024-12-04T14:57:36,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/7fc47ff91b8f45f58900e273dd5eee07 is 50, key is test_row_0/B:col10/1733324255710/Put/seqid=0 2024-12-04T14:57:36,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742152_1328 (size=9657) 2024-12-04T14:57:36,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324316865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324316865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324316975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324316975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324316975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:36,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/7fc47ff91b8f45f58900e273dd5eee07 2024-12-04T14:57:36,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/5c6247971bbe4966883b40940aad1955 is 50, key is test_row_0/C:col10/1733324255710/Put/seqid=0 2024-12-04T14:57:36,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742153_1329 (size=9657) 2024-12-04T14:57:37,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/5c6247971bbe4966883b40940aad1955 2024-12-04T14:57:37,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/f567531131564d0dbfb2ec3463b31745 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f567531131564d0dbfb2ec3463b31745 2024-12-04T14:57:37,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f567531131564d0dbfb2ec3463b31745, entries=150, sequenceid=94, filesize=30.2 K 2024-12-04T14:57:37,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/7fc47ff91b8f45f58900e273dd5eee07 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7fc47ff91b8f45f58900e273dd5eee07 2024-12-04T14:57:37,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7fc47ff91b8f45f58900e273dd5eee07, entries=100, sequenceid=94, filesize=9.4 K 2024-12-04T14:57:37,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/5c6247971bbe4966883b40940aad1955 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/5c6247971bbe4966883b40940aad1955 2024-12-04T14:57:37,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/5c6247971bbe4966883b40940aad1955, entries=100, sequenceid=94, filesize=9.4 K 2024-12-04T14:57:37,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0ed254c309e31d6514a1b829d83e0676 in 1662ms, sequenceid=94, compaction requested=true 2024-12-04T14:57:37,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:37,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:37,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:37,411 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:37,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:37,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:37,412 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:37,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:37,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:37,412 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:37,412 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:37,412 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/A is initiating minor compaction (all files) 2024-12-04T14:57:37,412 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/B is initiating minor compaction (all files) 2024-12-04T14:57:37,412 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/A in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:37,412 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/B in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:37,412 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7f40cd6d67fe412c924f8c31a7dd3839, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/fd6d4fbd4b754ebe96e1c946ca7b862c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f567531131564d0dbfb2ec3463b31745] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=90.8 K 2024-12-04T14:57:37,412 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/fe024119a6fb4063931b83b9d3c74a2b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/6d22ff237e5a458bb96293c43019f26a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7fc47ff91b8f45f58900e273dd5eee07] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=33.0 K 2024-12-04T14:57:37,412 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:37,413 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7f40cd6d67fe412c924f8c31a7dd3839, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/fd6d4fbd4b754ebe96e1c946ca7b862c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f567531131564d0dbfb2ec3463b31745] 2024-12-04T14:57:37,413 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting fe024119a6fb4063931b83b9d3c74a2b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733324251508 2024-12-04T14:57:37,413 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f40cd6d67fe412c924f8c31a7dd3839, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733324251508 2024-12-04T14:57:37,413 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d22ff237e5a458bb96293c43019f26a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324253596 2024-12-04T14:57:37,413 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd6d4fbd4b754ebe96e1c946ca7b862c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324253596 2024-12-04T14:57:37,413 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting f567531131564d0dbfb2ec3463b31745, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733324255710 2024-12-04T14:57:37,413 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fc47ff91b8f45f58900e273dd5eee07, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733324255710 2024-12-04T14:57:37,419 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:37,420 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/7af0a584b4784362b056c52bde018c58 is 50, key is test_row_0/B:col10/1733324255710/Put/seqid=0 2024-12-04T14:57:37,427 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:37,432 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120495efa3d5d183432eae3fffcf8b1eb2e1_0ed254c309e31d6514a1b829d83e0676 store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:37,433 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120495efa3d5d183432eae3fffcf8b1eb2e1_0ed254c309e31d6514a1b829d83e0676, store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:37,433 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120495efa3d5d183432eae3fffcf8b1eb2e1_0ed254c309e31d6514a1b829d83e0676 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:37,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742154_1330 (size=12207) 2024-12-04T14:57:37,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742155_1331 (size=4469) 2024-12-04T14:57:37,449 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#A#compaction#280 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:37,450 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/a7d5c11363544cfca8683505938e8fe2 is 175, key is test_row_0/A:col10/1733324255710/Put/seqid=0 2024-12-04T14:57:37,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742156_1332 (size=31268) 2024-12-04T14:57:37,455 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/7af0a584b4784362b056c52bde018c58 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7af0a584b4784362b056c52bde018c58 2024-12-04T14:57:37,461 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/B of 0ed254c309e31d6514a1b829d83e0676 into 7af0a584b4784362b056c52bde018c58(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:37,461 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:37,461 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/B, priority=13, startTime=1733324257411; duration=0sec 2024-12-04T14:57:37,461 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:37,461 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:B 2024-12-04T14:57:37,461 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:37,461 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/a7d5c11363544cfca8683505938e8fe2 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a7d5c11363544cfca8683505938e8fe2 2024-12-04T14:57:37,463 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:37,464 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/C is initiating minor compaction (all files) 2024-12-04T14:57:37,464 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/C in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:37,464 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0420f103d2c64cc688b2a4610b2f2e97, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/c89e3a9f06a448c1967756132856124b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/5c6247971bbe4966883b40940aad1955] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=33.0 K 2024-12-04T14:57:37,464 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 0420f103d2c64cc688b2a4610b2f2e97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733324251508 2024-12-04T14:57:37,465 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c89e3a9f06a448c1967756132856124b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324253596 2024-12-04T14:57:37,465 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c6247971bbe4966883b40940aad1955, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733324255710 2024-12-04T14:57:37,468 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/A of 0ed254c309e31d6514a1b829d83e0676 into a7d5c11363544cfca8683505938e8fe2(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:37,468 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:37,468 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/A, priority=13, startTime=1733324257411; duration=0sec 2024-12-04T14:57:37,468 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:37,468 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:A 2024-12-04T14:57:37,473 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#C#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:37,474 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/2a07b499ef8748fcadb8aed421d23f0d is 50, key is test_row_0/C:col10/1733324255710/Put/seqid=0 2024-12-04T14:57:37,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742157_1333 (size=12207) 2024-12-04T14:57:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T14:57:37,655 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:37,655 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-04T14:57:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T14:57:37,657 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:37,658 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:37,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T14:57:37,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T14:57:37,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:37,810 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-04T14:57:37,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048afa33e8fc474eb1b6a69d1148a29db8_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324255814/Put/seqid=0 2024-12-04T14:57:37,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742158_1334 (size=12154) 2024-12-04T14:57:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:37,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:37,882 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/2a07b499ef8748fcadb8aed421d23f0d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2a07b499ef8748fcadb8aed421d23f0d 2024-12-04T14:57:37,886 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/C of 0ed254c309e31d6514a1b829d83e0676 into 2a07b499ef8748fcadb8aed421d23f0d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:37,886 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:37,886 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/C, priority=13, startTime=1733324257412; duration=0sec 2024-12-04T14:57:37,886 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:37,886 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:C 2024-12-04T14:57:37,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324317903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:37,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324317908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T14:57:37,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324317995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:38,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324317995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:38,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324317995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:38,000 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:38,000 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:38,000 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:38,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324318015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:38,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324318015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:38,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:38,223 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048afa33e8fc474eb1b6a69d1148a29db8_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048afa33e8fc474eb1b6a69d1148a29db8_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:38,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/de88076fbe634acfb4862d6f2c64234a, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:38,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/de88076fbe634acfb4862d6f2c64234a is 175, key is test_row_0/A:col10/1733324255814/Put/seqid=0 2024-12-04T14:57:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742159_1335 (size=30955) 2024-12-04T14:57:38,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:38,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324318225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324318225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T14:57:38,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324318535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:38,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324318545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:38,630 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/de88076fbe634acfb4862d6f2c64234a 2024-12-04T14:57:38,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/64eb4a3ef0e04be6821df3bda801c79f is 50, key is test_row_0/B:col10/1733324255814/Put/seqid=0 2024-12-04T14:57:38,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742160_1336 (size=12001) 2024-12-04T14:57:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T14:57:39,040 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/64eb4a3ef0e04be6821df3bda801c79f 2024-12-04T14:57:39,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/d332bbb49ed74b82ab5e3cfe91f7c9e5 is 50, key is test_row_0/C:col10/1733324255814/Put/seqid=0 2024-12-04T14:57:39,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742161_1337 (size=12001) 2024-12-04T14:57:39,049 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/d332bbb49ed74b82ab5e3cfe91f7c9e5 2024-12-04T14:57:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324319045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/de88076fbe634acfb4862d6f2c64234a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/de88076fbe634acfb4862d6f2c64234a 2024-12-04T14:57:39,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324319055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:39,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:39,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:39,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:39,059 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/de88076fbe634acfb4862d6f2c64234a, entries=150, sequenceid=117, filesize=30.2 K 2024-12-04T14:57:39,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/64eb4a3ef0e04be6821df3bda801c79f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/64eb4a3ef0e04be6821df3bda801c79f 2024-12-04T14:57:39,064 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/64eb4a3ef0e04be6821df3bda801c79f, entries=150, sequenceid=117, filesize=11.7 K 2024-12-04T14:57:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/d332bbb49ed74b82ab5e3cfe91f7c9e5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/d332bbb49ed74b82ab5e3cfe91f7c9e5 2024-12-04T14:57:39,069 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/d332bbb49ed74b82ab5e3cfe91f7c9e5, entries=150, sequenceid=117, filesize=11.7 K 2024-12-04T14:57:39,070 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 0ed254c309e31d6514a1b829d83e0676 in 1260ms, sequenceid=117, compaction requested=false 2024-12-04T14:57:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-04T14:57:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-04T14:57:39,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-04T14:57:39,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-12-04T14:57:39,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.4190 sec 2024-12-04T14:57:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T14:57:39,805 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:39,805 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-04T14:57:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T14:57:39,808 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:39,809 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:39,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T14:57:39,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T14:57:39,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:39,961 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-04T14:57:39,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e266177aedee428a94259ac1e45af0f9_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324257896/Put/seqid=0 2024-12-04T14:57:39,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742162_1338 (size=12254) 2024-12-04T14:57:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:40,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324320049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324320051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324320052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324320065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,072 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324320075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,082 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T14:57:40,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324320165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324320165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324320165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:40,373 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e266177aedee428a94259ac1e45af0f9_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e266177aedee428a94259ac1e45af0f9_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:40,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/f5d364618c84492f946706f9d911939d, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/f5d364618c84492f946706f9d911939d is 175, key is test_row_0/A:col10/1733324257896/Put/seqid=0 2024-12-04T14:57:40,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742163_1339 (size=31055) 2024-12-04T14:57:40,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324320385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324320385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324320385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T14:57:40,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324320695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324320695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324320695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:40,779 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/f5d364618c84492f946706f9d911939d 2024-12-04T14:57:40,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/363ab5040b1946c8b2120bbfc0915d3b is 50, key is test_row_0/B:col10/1733324257896/Put/seqid=0 2024-12-04T14:57:40,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742164_1340 (size=12101) 2024-12-04T14:57:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T14:57:41,191 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/363ab5040b1946c8b2120bbfc0915d3b 2024-12-04T14:57:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/a9c4cf56dc39448e91a0865d0e092122 is 50, key is test_row_0/C:col10/1733324257896/Put/seqid=0 2024-12-04T14:57:41,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742165_1341 (size=12101) 2024-12-04T14:57:41,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324321205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:41,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324321205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:41,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324321205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:41,599 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/a9c4cf56dc39448e91a0865d0e092122 2024-12-04T14:57:41,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/f5d364618c84492f946706f9d911939d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f5d364618c84492f946706f9d911939d 2024-12-04T14:57:41,606 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f5d364618c84492f946706f9d911939d, entries=150, sequenceid=133, filesize=30.3 K 2024-12-04T14:57:41,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/363ab5040b1946c8b2120bbfc0915d3b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/363ab5040b1946c8b2120bbfc0915d3b 2024-12-04T14:57:41,609 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/363ab5040b1946c8b2120bbfc0915d3b, entries=150, sequenceid=133, filesize=11.8 K 2024-12-04T14:57:41,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/a9c4cf56dc39448e91a0865d0e092122 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/a9c4cf56dc39448e91a0865d0e092122 2024-12-04T14:57:41,612 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/a9c4cf56dc39448e91a0865d0e092122, entries=150, sequenceid=133, filesize=11.8 K 2024-12-04T14:57:41,613 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0ed254c309e31d6514a1b829d83e0676 in 1652ms, sequenceid=133, compaction requested=true 2024-12-04T14:57:41,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:41,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:41,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-04T14:57:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-04T14:57:41,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-04T14:57:41,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8060 sec 2024-12-04T14:57:41,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8120 sec 2024-12-04T14:57:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T14:57:41,955 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:41,955 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-04T14:57:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T14:57:41,957 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:41,957 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:41,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T14:57:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:42,089 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:42,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-04T14:57:42,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047a250536397a4784ad3ce6b9466c4097_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324260046/Put/seqid=0 2024-12-04T14:57:42,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742166_1342 (size=14794) 2024-12-04T14:57:42,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:42,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324322132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324322134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324322215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,220 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324322225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324322225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,232 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:42,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,232 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:42,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324322245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324322245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T14:57:42,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:42,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:42,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324322455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324322455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:42,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047a250536397a4784ad3ce6b9466c4097_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047a250536397a4784ad3ce6b9466c4097_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:42,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/5e6625c0c9fd4c458ed60f45dd059568, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:42,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/5e6625c0c9fd4c458ed60f45dd059568 is 175, key is test_row_0/A:col10/1733324260046/Put/seqid=0 2024-12-04T14:57:42,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742167_1343 (size=39749) 2024-12-04T14:57:42,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:42,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:42,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T14:57:42,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:42,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:42,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324322775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324322775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:42,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:42,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:42,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/5e6625c0c9fd4c458ed60f45dd059568 2024-12-04T14:57:42,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/ed5e6f88ef6344c792664ddd8e665526 is 50, key is test_row_0/B:col10/1733324260046/Put/seqid=0 2024-12-04T14:57:42,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742168_1344 (size=12151) 2024-12-04T14:57:43,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T14:57:43,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:43,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:43,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324323285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:43,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324323295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:43,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/ed5e6f88ef6344c792664ddd8e665526 2024-12-04T14:57:43,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/bebb44ed8aff4201b949a9f4fc03fd1c is 50, key is test_row_0/C:col10/1733324260046/Put/seqid=0 2024-12-04T14:57:43,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742169_1345 (size=12151) 2024-12-04T14:57:43,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:43,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:43,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:43,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/bebb44ed8aff4201b949a9f4fc03fd1c 2024-12-04T14:57:43,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/5e6625c0c9fd4c458ed60f45dd059568 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/5e6625c0c9fd4c458ed60f45dd059568 2024-12-04T14:57:43,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/5e6625c0c9fd4c458ed60f45dd059568, entries=200, sequenceid=155, filesize=38.8 K 2024-12-04T14:57:43,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/ed5e6f88ef6344c792664ddd8e665526 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/ed5e6f88ef6344c792664ddd8e665526 2024-12-04T14:57:43,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/ed5e6f88ef6344c792664ddd8e665526, entries=150, sequenceid=155, filesize=11.9 K 2024-12-04T14:57:43,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/bebb44ed8aff4201b949a9f4fc03fd1c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/bebb44ed8aff4201b949a9f4fc03fd1c 2024-12-04T14:57:43,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/bebb44ed8aff4201b949a9f4fc03fd1c, entries=150, sequenceid=155, filesize=11.9 K 2024-12-04T14:57:43,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0ed254c309e31d6514a1b829d83e0676 in 1650ms, sequenceid=155, compaction requested=true 2024-12-04T14:57:43,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:43,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:43,739 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:57:43,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:43,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:43,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:43,739 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:57:43,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:43,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:43,740 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133027 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:57:43,740 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:57:43,740 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/B is initiating minor compaction (all files) 2024-12-04T14:57:43,740 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/A is initiating minor compaction (all files) 2024-12-04T14:57:43,740 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/A in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,740 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/B in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,740 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a7d5c11363544cfca8683505938e8fe2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/de88076fbe634acfb4862d6f2c64234a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f5d364618c84492f946706f9d911939d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/5e6625c0c9fd4c458ed60f45dd059568] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=129.9 K 2024-12-04T14:57:43,740 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7af0a584b4784362b056c52bde018c58, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/64eb4a3ef0e04be6821df3bda801c79f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/363ab5040b1946c8b2120bbfc0915d3b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/ed5e6f88ef6344c792664ddd8e665526] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=47.3 K 2024-12-04T14:57:43,740 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,740 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a7d5c11363544cfca8683505938e8fe2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/de88076fbe634acfb4862d6f2c64234a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f5d364618c84492f946706f9d911939d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/5e6625c0c9fd4c458ed60f45dd059568] 2024-12-04T14:57:43,741 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af0a584b4784362b056c52bde018c58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733324253609 2024-12-04T14:57:43,741 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d5c11363544cfca8683505938e8fe2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733324253609 2024-12-04T14:57:43,741 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 64eb4a3ef0e04be6821df3bda801c79f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324255814 2024-12-04T14:57:43,741 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting de88076fbe634acfb4862d6f2c64234a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324255814 2024-12-04T14:57:43,741 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 363ab5040b1946c8b2120bbfc0915d3b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733324257896 2024-12-04T14:57:43,741 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting ed5e6f88ef6344c792664ddd8e665526, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733324260046 2024-12-04T14:57:43,741 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5d364618c84492f946706f9d911939d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733324257896 2024-12-04T14:57:43,742 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e6625c0c9fd4c458ed60f45dd059568, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733324260046 2024-12-04T14:57:43,751 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:43,751 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/767f38054f434069863760065bc39944 is 50, key is test_row_0/B:col10/1733324260046/Put/seqid=0 2024-12-04T14:57:43,753 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:43,755 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412047a9829eede934726baf6fbb0c83d9499_0ed254c309e31d6514a1b829d83e0676 store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:43,757 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412047a9829eede934726baf6fbb0c83d9499_0ed254c309e31d6514a1b829d83e0676, store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:43,757 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047a9829eede934726baf6fbb0c83d9499_0ed254c309e31d6514a1b829d83e0676 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:43,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742170_1346 (size=12493) 2024-12-04T14:57:43,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742171_1347 (size=4469) 2024-12-04T14:57:43,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T14:57:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:43,792 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-04T14:57:43,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c666942a191b40e2813270eea39b8847_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324262133/Put/seqid=0 2024-12-04T14:57:43,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742172_1348 (size=12304) 2024-12-04T14:57:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T14:57:44,163 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/767f38054f434069863760065bc39944 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/767f38054f434069863760065bc39944 2024-12-04T14:57:44,165 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#A#compaction#292 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:44,165 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/d5bacffdf6924db69caa075243c69853 is 175, key is test_row_0/A:col10/1733324260046/Put/seqid=0 2024-12-04T14:57:44,167 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/B of 0ed254c309e31d6514a1b829d83e0676 into 767f38054f434069863760065bc39944(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:44,167 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:44,167 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/B, priority=12, startTime=1733324263739; duration=0sec 2024-12-04T14:57:44,167 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:44,167 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:B 2024-12-04T14:57:44,167 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:57:44,168 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:57:44,168 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/C is initiating minor compaction (all files) 2024-12-04T14:57:44,168 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/C in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:44,169 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2a07b499ef8748fcadb8aed421d23f0d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/d332bbb49ed74b82ab5e3cfe91f7c9e5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/a9c4cf56dc39448e91a0865d0e092122, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/bebb44ed8aff4201b949a9f4fc03fd1c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=47.3 K 2024-12-04T14:57:44,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742173_1349 (size=31447) 2024-12-04T14:57:44,169 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a07b499ef8748fcadb8aed421d23f0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733324253609 2024-12-04T14:57:44,170 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting d332bbb49ed74b82ab5e3cfe91f7c9e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324255814 2024-12-04T14:57:44,170 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c4cf56dc39448e91a0865d0e092122, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733324257896 2024-12-04T14:57:44,170 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting bebb44ed8aff4201b949a9f4fc03fd1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733324260046 2024-12-04T14:57:44,175 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/d5bacffdf6924db69caa075243c69853 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/d5bacffdf6924db69caa075243c69853 2024-12-04T14:57:44,179 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:44,180 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/A of 0ed254c309e31d6514a1b829d83e0676 into d5bacffdf6924db69caa075243c69853(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:44,180 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:44,180 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/b8f3dede4854438f8803a3b6ccec508a is 50, key is test_row_0/C:col10/1733324260046/Put/seqid=0 2024-12-04T14:57:44,180 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/A, priority=12, startTime=1733324263739; duration=0sec 2024-12-04T14:57:44,180 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:44,180 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:A 2024-12-04T14:57:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742174_1350 (size=12493) 2024-12-04T14:57:44,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:44,206 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c666942a191b40e2813270eea39b8847_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c666942a191b40e2813270eea39b8847_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:44,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/14541577da2e462992482ca2ee72e5df, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:44,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/14541577da2e462992482ca2ee72e5df is 175, key is test_row_0/A:col10/1733324262133/Put/seqid=0 2024-12-04T14:57:44,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742175_1351 (size=31105) 2024-12-04T14:57:44,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:44,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324324288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324324297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324324297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324324305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,310 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324324315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,320 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324324405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324324405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324324405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,592 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/b8f3dede4854438f8803a3b6ccec508a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/b8f3dede4854438f8803a3b6ccec508a 2024-12-04T14:57:44,599 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/C of 0ed254c309e31d6514a1b829d83e0676 into b8f3dede4854438f8803a3b6ccec508a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:44,599 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:44,599 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/C, priority=12, startTime=1733324263739; duration=0sec 2024-12-04T14:57:44,599 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:44,599 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:C 2024-12-04T14:57:44,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324324615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324324615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324324615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,621 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/14541577da2e462992482ca2ee72e5df 2024-12-04T14:57:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/992edc3d01934a15a25811af9ef6e5af is 50, key is test_row_0/B:col10/1733324262133/Put/seqid=0 2024-12-04T14:57:44,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742176_1352 (size=12151) 2024-12-04T14:57:44,650 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/992edc3d01934a15a25811af9ef6e5af 2024-12-04T14:57:44,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/2f168624c41a4e249a839129aa515a47 is 50, key is test_row_0/C:col10/1733324262133/Put/seqid=0 2024-12-04T14:57:44,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742177_1353 (size=12151) 2024-12-04T14:57:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324324925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324324925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324324925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:45,070 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/2f168624c41a4e249a839129aa515a47 2024-12-04T14:57:45,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/14541577da2e462992482ca2ee72e5df as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/14541577da2e462992482ca2ee72e5df 2024-12-04T14:57:45,082 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/14541577da2e462992482ca2ee72e5df, entries=150, sequenceid=169, filesize=30.4 K 2024-12-04T14:57:45,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/992edc3d01934a15a25811af9ef6e5af as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/992edc3d01934a15a25811af9ef6e5af 2024-12-04T14:57:45,087 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/992edc3d01934a15a25811af9ef6e5af, entries=150, sequenceid=169, filesize=11.9 K 2024-12-04T14:57:45,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/2f168624c41a4e249a839129aa515a47 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2f168624c41a4e249a839129aa515a47 2024-12-04T14:57:45,092 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2f168624c41a4e249a839129aa515a47, entries=150, sequenceid=169, filesize=11.9 K 2024-12-04T14:57:45,093 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 0ed254c309e31d6514a1b829d83e0676 in 1301ms, sequenceid=169, compaction requested=false 2024-12-04T14:57:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-04T14:57:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-04T14:57:45,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-04T14:57:45,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1380 sec 2024-12-04T14:57:45,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.1430 sec 2024-12-04T14:57:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:45,438 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:45,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-04T14:57:45,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120428b86834852d44b5a9669e0460aba9ab_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324264287/Put/seqid=0 2024-12-04T14:57:45,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742178_1354 (size=14794) 2024-12-04T14:57:45,452 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:45,455 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120428b86834852d44b5a9669e0460aba9ab_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120428b86834852d44b5a9669e0460aba9ab_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:45,456 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/2cb7f0edbab74235beeae4b829be379c, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:45,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/2cb7f0edbab74235beeae4b829be379c is 175, key is test_row_0/A:col10/1733324264287/Put/seqid=0 2024-12-04T14:57:45,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742179_1355 (size=39749) 2024-12-04T14:57:45,466 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/2cb7f0edbab74235beeae4b829be379c 2024-12-04T14:57:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324325459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:45,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/05f13af59fd24191aa40eabe28b832f2 is 50, key is test_row_0/B:col10/1733324264287/Put/seqid=0 2024-12-04T14:57:45,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324325467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:45,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324325467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:45,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:45,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742180_1356 (size=12151) 2024-12-04T14:57:45,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324325574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:45,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:45,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324325585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:45,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324325585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:45,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324325785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:45,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324325794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324325795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:45,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/05f13af59fd24191aa40eabe28b832f2 2024-12-04T14:57:45,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/ba169fb190d841be81a0f2a44e075bc4 is 50, key is test_row_0/C:col10/1733324264287/Put/seqid=0 2024-12-04T14:57:45,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742181_1357 (size=12151) 2024-12-04T14:57:46,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324326095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324326105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324326105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T14:57:46,135 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:46,136 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-04T14:57:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T14:57:46,138 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:46,139 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:46,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T14:57:46,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/ba169fb190d841be81a0f2a44e075bc4 2024-12-04T14:57:46,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:46,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:46,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/2cb7f0edbab74235beeae4b829be379c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/2cb7f0edbab74235beeae4b829be379c 2024-12-04T14:57:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/2cb7f0edbab74235beeae4b829be379c, entries=200, sequenceid=195, filesize=38.8 K 2024-12-04T14:57:46,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/05f13af59fd24191aa40eabe28b832f2 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/05f13af59fd24191aa40eabe28b832f2 2024-12-04T14:57:46,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/05f13af59fd24191aa40eabe28b832f2, entries=150, sequenceid=195, filesize=11.9 K 2024-12-04T14:57:46,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/ba169fb190d841be81a0f2a44e075bc4 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ba169fb190d841be81a0f2a44e075bc4 2024-12-04T14:57:46,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ba169fb190d841be81a0f2a44e075bc4, entries=150, sequenceid=195, filesize=11.9 K 2024-12-04T14:57:46,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 0ed254c309e31d6514a1b829d83e0676 in 881ms, sequenceid=195, compaction requested=true 2024-12-04T14:57:46,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:46,320 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:46,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:46,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:46,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:46,320 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:46,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:46,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:46,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:46,321 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:46,321 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/A is initiating minor compaction (all files) 2024-12-04T14:57:46,321 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/A in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,321 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/d5bacffdf6924db69caa075243c69853, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/14541577da2e462992482ca2ee72e5df, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/2cb7f0edbab74235beeae4b829be379c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=99.9 K 2024-12-04T14:57:46,321 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,321 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/d5bacffdf6924db69caa075243c69853, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/14541577da2e462992482ca2ee72e5df, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/2cb7f0edbab74235beeae4b829be379c] 2024-12-04T14:57:46,321 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:46,321 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/B is initiating minor compaction (all files) 2024-12-04T14:57:46,321 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5bacffdf6924db69caa075243c69853, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733324260046 2024-12-04T14:57:46,321 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/B in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,321 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/767f38054f434069863760065bc39944, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/992edc3d01934a15a25811af9ef6e5af, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/05f13af59fd24191aa40eabe28b832f2] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=35.9 K 2024-12-04T14:57:46,322 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14541577da2e462992482ca2ee72e5df, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733324262120 2024-12-04T14:57:46,322 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 767f38054f434069863760065bc39944, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733324260046 2024-12-04T14:57:46,322 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cb7f0edbab74235beeae4b829be379c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733324264286 2024-12-04T14:57:46,322 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 992edc3d01934a15a25811af9ef6e5af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733324262120 2024-12-04T14:57:46,322 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f13af59fd24191aa40eabe28b832f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733324264287 2024-12-04T14:57:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:46,330 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:46,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-04T14:57:46,342 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:46,343 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/9da45a995a13414380918fbf4e8e6d20 is 50, key is test_row_0/B:col10/1733324264287/Put/seqid=0 2024-12-04T14:57:46,344 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:46,359 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204100f0e1ec41a42c5b5eb0d84f536294d_0ed254c309e31d6514a1b829d83e0676 store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:46,361 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204100f0e1ec41a42c5b5eb0d84f536294d_0ed254c309e31d6514a1b829d83e0676, store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:46,361 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204100f0e1ec41a42c5b5eb0d84f536294d_0ed254c309e31d6514a1b829d83e0676 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:46,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742182_1358 (size=12595) 2024-12-04T14:57:46,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120465652ef3156f4613ac3f031c7f1b8492_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324266329/Put/seqid=0 2024-12-04T14:57:46,387 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/9da45a995a13414380918fbf4e8e6d20 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/9da45a995a13414380918fbf4e8e6d20 2024-12-04T14:57:46,392 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/B of 0ed254c309e31d6514a1b829d83e0676 into 9da45a995a13414380918fbf4e8e6d20(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:46,392 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:46,392 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/B, priority=13, startTime=1733324266320; duration=0sec 2024-12-04T14:57:46,392 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:46,393 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:B 2024-12-04T14:57:46,393 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:46,394 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:46,394 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 0ed254c309e31d6514a1b829d83e0676/C is initiating minor compaction (all files) 2024-12-04T14:57:46,394 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ed254c309e31d6514a1b829d83e0676/C in TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,394 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/b8f3dede4854438f8803a3b6ccec508a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2f168624c41a4e249a839129aa515a47, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ba169fb190d841be81a0f2a44e075bc4] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp, totalSize=35.9 K 2024-12-04T14:57:46,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742183_1359 (size=4469) 2024-12-04T14:57:46,406 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f3dede4854438f8803a3b6ccec508a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733324260046 2024-12-04T14:57:46,408 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f168624c41a4e249a839129aa515a47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733324262120 2024-12-04T14:57:46,409 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting ba169fb190d841be81a0f2a44e075bc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733324264287 2024-12-04T14:57:46,412 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#A#compaction#301 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:46,412 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/3940130b18e544c99926fa73fdb11322 is 175, key is test_row_0/A:col10/1733324264287/Put/seqid=0 2024-12-04T14:57:46,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742184_1360 (size=14794) 2024-12-04T14:57:46,425 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:46,431 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120465652ef3156f4613ac3f031c7f1b8492_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465652ef3156f4613ac3f031c7f1b8492_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:46,432 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/a76f5c98194b44e88f2bfc84db80dbfb, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:46,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/a76f5c98194b44e88f2bfc84db80dbfb is 175, key is test_row_0/A:col10/1733324266329/Put/seqid=0 2024-12-04T14:57:46,436 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ed254c309e31d6514a1b829d83e0676#C#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:46,436 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/ddc33ca054854b0ba41b8908128c0e72 is 50, key is test_row_0/C:col10/1733324264287/Put/seqid=0 2024-12-04T14:57:46,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:46,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324326439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324326439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:46,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742185_1361 (size=31549) 2024-12-04T14:57:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T14:57:46,459 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/3940130b18e544c99926fa73fdb11322 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/3940130b18e544c99926fa73fdb11322 2024-12-04T14:57:46,464 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/A of 0ed254c309e31d6514a1b829d83e0676 into 3940130b18e544c99926fa73fdb11322(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:46,464 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:46,464 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/A, priority=13, startTime=1733324266320; duration=0sec 2024-12-04T14:57:46,464 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:46,464 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:A 2024-12-04T14:57:46,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742186_1362 (size=39749) 2024-12-04T14:57:46,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742187_1363 (size=12595) 2024-12-04T14:57:46,479 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/ddc33ca054854b0ba41b8908128c0e72 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ddc33ca054854b0ba41b8908128c0e72 2024-12-04T14:57:46,485 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ed254c309e31d6514a1b829d83e0676/C of 0ed254c309e31d6514a1b829d83e0676 into ddc33ca054854b0ba41b8908128c0e72(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:46,485 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:46,485 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., storeName=0ed254c309e31d6514a1b829d83e0676/C, priority=13, startTime=1733324266320; duration=0sec 2024-12-04T14:57:46,485 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:46,485 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:C 2024-12-04T14:57:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324326556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324326556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:46,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:46,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324326605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324326615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324326615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T14:57:46,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324326765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324326776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:46,869 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/a76f5c98194b44e88f2bfc84db80dbfb 2024-12-04T14:57:46,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/f3831a52d2d84fdeb3fec5cd2fbc0d1b is 50, key is test_row_0/B:col10/1733324266329/Put/seqid=0 2024-12-04T14:57:46,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:46,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:46,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742188_1364 (size=12151) 2024-12-04T14:57:47,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324327075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:47,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324327095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:47,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:47,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:47,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T14:57:47,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/f3831a52d2d84fdeb3fec5cd2fbc0d1b 2024-12-04T14:57:47,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/1b6233bd72f14499bdccb3cbd548ffad is 50, key is test_row_0/C:col10/1733324266329/Put/seqid=0 2024-12-04T14:57:47,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742189_1365 (size=12151) 2024-12-04T14:57:47,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:47,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:47,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:47,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:47,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324327586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:47,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:47,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324327617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:47,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37396 deadline: 1733324327626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:47,635 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:47,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37466 deadline: 1733324327637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:47,640 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:47,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37444 deadline: 1733324327637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:47,642 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 0ed254c309e31d6514a1b829d83e0676 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:57:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:47,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:47,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:47,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:47,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/1b6233bd72f14499bdccb3cbd548ffad 2024-12-04T14:57:47,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/a76f5c98194b44e88f2bfc84db80dbfb as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a76f5c98194b44e88f2bfc84db80dbfb 2024-12-04T14:57:47,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a76f5c98194b44e88f2bfc84db80dbfb, entries=200, sequenceid=210, filesize=38.8 K 2024-12-04T14:57:47,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/f3831a52d2d84fdeb3fec5cd2fbc0d1b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/f3831a52d2d84fdeb3fec5cd2fbc0d1b 2024-12-04T14:57:47,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/f3831a52d2d84fdeb3fec5cd2fbc0d1b, entries=150, sequenceid=210, filesize=11.9 K 2024-12-04T14:57:47,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/1b6233bd72f14499bdccb3cbd548ffad as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/1b6233bd72f14499bdccb3cbd548ffad 2024-12-04T14:57:47,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/1b6233bd72f14499bdccb3cbd548ffad, entries=150, sequenceid=210, filesize=11.9 K 2024-12-04T14:57:47,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 0ed254c309e31d6514a1b829d83e0676 in 1466ms, sequenceid=210, compaction requested=false 2024-12-04T14:57:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:47,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T14:57:47,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:47,833 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-04T14:57:47,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120498a229a6c4bd421dabb4c8768523d664_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324266437/Put/seqid=0 2024-12-04T14:57:47,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742190_1366 (size=12304) 2024-12-04T14:57:48,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:48,267 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120498a229a6c4bd421dabb4c8768523d664_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120498a229a6c4bd421dabb4c8768523d664_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:48,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/cdf5fd562c2f422eae8cfef1c784d5f4, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:48,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/cdf5fd562c2f422eae8cfef1c784d5f4 is 175, key is test_row_0/A:col10/1733324266437/Put/seqid=0 2024-12-04T14:57:48,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742191_1367 (size=31105) 2024-12-04T14:57:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T14:57:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:48,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. as already flushing 2024-12-04T14:57:48,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324328649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:48,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324328656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:48,679 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/cdf5fd562c2f422eae8cfef1c784d5f4 2024-12-04T14:57:48,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/e35fc8bf424c44c0848610e542ec39a9 is 50, key is test_row_0/B:col10/1733324266437/Put/seqid=0 2024-12-04T14:57:48,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742192_1368 (size=12151) 2024-12-04T14:57:48,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324328765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:48,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324328765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:48,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324328975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324328975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:49,099 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/e35fc8bf424c44c0848610e542ec39a9 2024-12-04T14:57:49,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/704dad97a6ac4b08b7236cac55f491e3 is 50, key is test_row_0/C:col10/1733324266437/Put/seqid=0 2024-12-04T14:57:49,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742193_1369 (size=12151) 2024-12-04T14:57:49,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37436 deadline: 1733324329285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:49,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37452 deadline: 1733324329285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:57:49,305 INFO [Thread-1425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1425. 2024-12-04T14:57:49,305 DEBUG [Thread-1425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:49,305 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,305 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:49,307 INFO [Thread-1423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1423. 2024-12-04T14:57:49,307 DEBUG [Thread-1423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:49,307 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,307 INFO [Thread-1417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1417. 2024-12-04T14:57:49,307 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,307 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:49,307 DEBUG [Thread-1417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:49,307 DEBUG [Thread-1417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,307 DEBUG [Thread-1417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,307 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:49,313 INFO [Thread-1421 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1421. 2024-12-04T14:57:49,313 DEBUG [Thread-1421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:49,313 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,313 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,313 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:49,314 INFO [Thread-1419 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1419. 2024-12-04T14:57:49,315 DEBUG [Thread-1419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:49,315 DEBUG [Thread-1419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,315 DEBUG [Thread-1419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:49,517 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/704dad97a6ac4b08b7236cac55f491e3 2024-12-04T14:57:49,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/cdf5fd562c2f422eae8cfef1c784d5f4 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/cdf5fd562c2f422eae8cfef1c784d5f4 2024-12-04T14:57:49,524 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/cdf5fd562c2f422eae8cfef1c784d5f4, entries=150, sequenceid=235, filesize=30.4 K 2024-12-04T14:57:49,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/e35fc8bf424c44c0848610e542ec39a9 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/e35fc8bf424c44c0848610e542ec39a9 2024-12-04T14:57:49,528 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/e35fc8bf424c44c0848610e542ec39a9, entries=150, sequenceid=235, filesize=11.9 K 2024-12-04T14:57:49,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/704dad97a6ac4b08b7236cac55f491e3 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/704dad97a6ac4b08b7236cac55f491e3 2024-12-04T14:57:49,532 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/704dad97a6ac4b08b7236cac55f491e3, entries=150, sequenceid=235, filesize=11.9 K 2024-12-04T14:57:49,533 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0ed254c309e31d6514a1b829d83e0676 in 1700ms, sequenceid=235, compaction requested=true 2024-12-04T14:57:49,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:49,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:49,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-04T14:57:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-04T14:57:49,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-04T14:57:49,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3950 sec 2024-12-04T14:57:49,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.4000 sec 2024-12-04T14:57:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:49,647 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:49,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-04T14:57:49,647 INFO [Thread-1408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1408. 2024-12-04T14:57:49,647 DEBUG [Thread-1408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:49,647 DEBUG [Thread-1408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,647 DEBUG [Thread-1408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:49,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041e61fce87c564e58abd8815399f7d21f_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324269645/Put/seqid=0 2024-12-04T14:57:49,656 INFO [Thread-1406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1406. 2024-12-04T14:57:49,656 INFO [Thread-1414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1414. 2024-12-04T14:57:49,656 DEBUG [Thread-1406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:49,656 DEBUG [Thread-1414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:49,656 DEBUG [Thread-1406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,656 DEBUG [Thread-1414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,656 DEBUG [Thread-1406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,656 DEBUG [Thread-1414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:49,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:49,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742194_1370 (size=12304) 2024-12-04T14:57:49,807 INFO [Thread-1412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1412. 2024-12-04T14:57:49,807 INFO [Thread-1410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1410. 2024-12-04T14:57:49,807 DEBUG [Thread-1410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:49,807 DEBUG [Thread-1412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:57:49,807 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,807 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,807 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,807 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:49,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:49,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:50,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:50,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041e61fce87c564e58abd8815399f7d21f_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041e61fce87c564e58abd8815399f7d21f_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:50,064 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/63f6e435bc9d4ed9b189c5e2461372d4, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:50,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/63f6e435bc9d4ed9b189c5e2461372d4 is 175, key is test_row_0/A:col10/1733324269645/Put/seqid=0 2024-12-04T14:57:50,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742195_1371 (size=31105) 2024-12-04T14:57:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T14:57:50,306 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1899 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5694 rows 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1889 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5661 rows 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1897 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5680 rows 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1889 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5667 rows 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1891 2024-12-04T14:57:50,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5668 rows 2024-12-04T14:57:50,306 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:57:50,306 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:57:50,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:50,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:50,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:57:50,307 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T14:57:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:50,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T14:57:50,310 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324270310"}]},"ts":"1733324270310"} 2024-12-04T14:57:50,312 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T14:57:50,312 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T14:57:50,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:57:50,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, UNASSIGN}] 2024-12-04T14:57:50,315 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, UNASSIGN 2024-12-04T14:57:50,316 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=0ed254c309e31d6514a1b829d83e0676, regionState=CLOSING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:50,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, UNASSIGN because future has completed 2024-12-04T14:57:50,317 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:57:50,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:57:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T14:57:50,469 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:50,469 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:57:50,469 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 0ed254c309e31d6514a1b829d83e0676, disabling compactions & flushes 2024-12-04T14:57:50,469 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/63f6e435bc9d4ed9b189c5e2461372d4 2024-12-04T14:57:50,470 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:50,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/ac855a0fe3ef47e8b4f58af3a03f457c is 50, key is test_row_0/B:col10/1733324269645/Put/seqid=0 2024-12-04T14:57:50,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742196_1372 (size=12151) 2024-12-04T14:57:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T14:57:50,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/ac855a0fe3ef47e8b4f58af3a03f457c 2024-12-04T14:57:50,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/ebd91f443ae64e8f88af795f4c5ab259 is 50, key is test_row_0/C:col10/1733324269645/Put/seqid=0 2024-12-04T14:57:50,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742197_1373 (size=12151) 2024-12-04T14:57:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T14:57:51,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/ebd91f443ae64e8f88af795f4c5ab259 2024-12-04T14:57:51,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/63f6e435bc9d4ed9b189c5e2461372d4 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/63f6e435bc9d4ed9b189c5e2461372d4 2024-12-04T14:57:51,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/63f6e435bc9d4ed9b189c5e2461372d4, entries=150, sequenceid=250, filesize=30.4 K 2024-12-04T14:57:51,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/ac855a0fe3ef47e8b4f58af3a03f457c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/ac855a0fe3ef47e8b4f58af3a03f457c 2024-12-04T14:57:51,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/ac855a0fe3ef47e8b4f58af3a03f457c, entries=150, sequenceid=250, filesize=11.9 K 2024-12-04T14:57:51,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/ebd91f443ae64e8f88af795f4c5ab259 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ebd91f443ae64e8f88af795f4c5ab259 2024-12-04T14:57:51,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ebd91f443ae64e8f88af795f4c5ab259, entries=150, sequenceid=250, filesize=11.9 K 2024-12-04T14:57:51,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=26.84 KB/27480 for 0ed254c309e31d6514a1b829d83e0676 in 1658ms, sequenceid=250, compaction requested=true 2024-12-04T14:57:51,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ed254c309e31d6514a1b829d83e0676: 2024-12-04T14:57:51,305 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:51,305 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:51,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:51,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:51,305 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. after waiting 0 ms 2024-12-04T14:57:51,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:51,305 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:51,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:51,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ed254c309e31d6514a1b829d83e0676:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:51,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:51,305 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing 0ed254c309e31d6514a1b829d83e0676 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-04T14:57:51,305 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. because compaction request was cancelled 2024-12-04T14:57:51,305 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. because compaction request was cancelled 2024-12-04T14:57:51,305 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:A 2024-12-04T14:57:51,305 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:B 2024-12-04T14:57:51,305 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. because compaction request was cancelled 2024-12-04T14:57:51,305 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ed254c309e31d6514a1b829d83e0676:C 2024-12-04T14:57:51,310 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040f3e12440f824dc8a582055a8800201b_0ed254c309e31d6514a1b829d83e0676 is 50, key is test_row_0/A:col10/1733324269805/Put/seqid=0 2024-12-04T14:57:51,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742198_1374 (size=9814) 2024-12-04T14:57:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T14:57:51,714 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:51,717 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040f3e12440f824dc8a582055a8800201b_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040f3e12440f824dc8a582055a8800201b_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:51,718 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/ae0bcb0fe5294be1b4978860119635e3, store: [table=TestAcidGuarantees family=A region=0ed254c309e31d6514a1b829d83e0676] 2024-12-04T14:57:51,719 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/ae0bcb0fe5294be1b4978860119635e3 is 175, key is test_row_0/A:col10/1733324269805/Put/seqid=0 2024-12-04T14:57:51,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742199_1375 (size=22461) 2024-12-04T14:57:52,122 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/ae0bcb0fe5294be1b4978860119635e3 2024-12-04T14:57:52,128 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/4aa8257c725c4e6aa05f6dd93ce1bb1d is 50, key is test_row_0/B:col10/1733324269805/Put/seqid=0 2024-12-04T14:57:52,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742200_1376 (size=9757) 2024-12-04T14:57:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T14:57:52,534 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/4aa8257c725c4e6aa05f6dd93ce1bb1d 2024-12-04T14:57:52,541 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/c1fecc78cfe84aa08f1420731f2400fd is 50, key is test_row_0/C:col10/1733324269805/Put/seqid=0 2024-12-04T14:57:52,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742201_1377 (size=9757) 2024-12-04T14:57:52,946 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/c1fecc78cfe84aa08f1420731f2400fd 2024-12-04T14:57:52,955 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/A/ae0bcb0fe5294be1b4978860119635e3 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/ae0bcb0fe5294be1b4978860119635e3 2024-12-04T14:57:52,961 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/ae0bcb0fe5294be1b4978860119635e3, entries=100, sequenceid=257, filesize=21.9 K 2024-12-04T14:57:52,962 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/B/4aa8257c725c4e6aa05f6dd93ce1bb1d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/4aa8257c725c4e6aa05f6dd93ce1bb1d 2024-12-04T14:57:52,966 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/4aa8257c725c4e6aa05f6dd93ce1bb1d, entries=100, sequenceid=257, filesize=9.5 K 2024-12-04T14:57:52,967 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/.tmp/C/c1fecc78cfe84aa08f1420731f2400fd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/c1fecc78cfe84aa08f1420731f2400fd 2024-12-04T14:57:52,970 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/c1fecc78cfe84aa08f1420731f2400fd, entries=100, sequenceid=257, filesize=9.5 K 2024-12-04T14:57:52,971 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0ed254c309e31d6514a1b829d83e0676 in 1666ms, sequenceid=257, compaction requested=true 2024-12-04T14:57:52,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a9d32e00b46042909cbf9816eb948576, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7c6f26e6d723417b9252a9cdd762ca36, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7f40cd6d67fe412c924f8c31a7dd3839, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/c33310f6384242d4886a21a6ab7899a1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/fd6d4fbd4b754ebe96e1c946ca7b862c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a7d5c11363544cfca8683505938e8fe2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f567531131564d0dbfb2ec3463b31745, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/de88076fbe634acfb4862d6f2c64234a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f5d364618c84492f946706f9d911939d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/5e6625c0c9fd4c458ed60f45dd059568, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/d5bacffdf6924db69caa075243c69853, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/14541577da2e462992482ca2ee72e5df, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/2cb7f0edbab74235beeae4b829be379c] to archive 2024-12-04T14:57:52,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:57:52,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a9d32e00b46042909cbf9816eb948576 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a9d32e00b46042909cbf9816eb948576 2024-12-04T14:57:52,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7c6f26e6d723417b9252a9cdd762ca36 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7c6f26e6d723417b9252a9cdd762ca36 2024-12-04T14:57:52,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7f40cd6d67fe412c924f8c31a7dd3839 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/7f40cd6d67fe412c924f8c31a7dd3839 2024-12-04T14:57:52,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/c33310f6384242d4886a21a6ab7899a1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/c33310f6384242d4886a21a6ab7899a1 2024-12-04T14:57:52,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/fd6d4fbd4b754ebe96e1c946ca7b862c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/fd6d4fbd4b754ebe96e1c946ca7b862c 2024-12-04T14:57:52,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a7d5c11363544cfca8683505938e8fe2 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a7d5c11363544cfca8683505938e8fe2 2024-12-04T14:57:52,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f567531131564d0dbfb2ec3463b31745 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f567531131564d0dbfb2ec3463b31745 2024-12-04T14:57:52,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/de88076fbe634acfb4862d6f2c64234a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/de88076fbe634acfb4862d6f2c64234a 2024-12-04T14:57:52,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f5d364618c84492f946706f9d911939d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/f5d364618c84492f946706f9d911939d 2024-12-04T14:57:52,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/5e6625c0c9fd4c458ed60f45dd059568 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/5e6625c0c9fd4c458ed60f45dd059568 2024-12-04T14:57:52,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/d5bacffdf6924db69caa075243c69853 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/d5bacffdf6924db69caa075243c69853 2024-12-04T14:57:52,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/14541577da2e462992482ca2ee72e5df to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/14541577da2e462992482ca2ee72e5df 2024-12-04T14:57:52,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/2cb7f0edbab74235beeae4b829be379c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/2cb7f0edbab74235beeae4b829be379c 2024-12-04T14:57:52,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/b00f182635b140e4a9bf394f77a769b4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/9f6d4231663b437b916f8faabdc4d509, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/fe024119a6fb4063931b83b9d3c74a2b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/21a9f0cc3cd24921933b025a979bf6a5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/6d22ff237e5a458bb96293c43019f26a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7af0a584b4784362b056c52bde018c58, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7fc47ff91b8f45f58900e273dd5eee07, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/64eb4a3ef0e04be6821df3bda801c79f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/363ab5040b1946c8b2120bbfc0915d3b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/767f38054f434069863760065bc39944, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/ed5e6f88ef6344c792664ddd8e665526, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/992edc3d01934a15a25811af9ef6e5af, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/05f13af59fd24191aa40eabe28b832f2] to archive 2024-12-04T14:57:52,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:57:52,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/b00f182635b140e4a9bf394f77a769b4 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/b00f182635b140e4a9bf394f77a769b4 2024-12-04T14:57:52,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/9f6d4231663b437b916f8faabdc4d509 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/9f6d4231663b437b916f8faabdc4d509 2024-12-04T14:57:52,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/fe024119a6fb4063931b83b9d3c74a2b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/fe024119a6fb4063931b83b9d3c74a2b 2024-12-04T14:57:52,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/21a9f0cc3cd24921933b025a979bf6a5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/21a9f0cc3cd24921933b025a979bf6a5 2024-12-04T14:57:52,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/6d22ff237e5a458bb96293c43019f26a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/6d22ff237e5a458bb96293c43019f26a 2024-12-04T14:57:52,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7af0a584b4784362b056c52bde018c58 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7af0a584b4784362b056c52bde018c58 2024-12-04T14:57:52,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7fc47ff91b8f45f58900e273dd5eee07 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/7fc47ff91b8f45f58900e273dd5eee07 2024-12-04T14:57:52,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/64eb4a3ef0e04be6821df3bda801c79f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/64eb4a3ef0e04be6821df3bda801c79f 2024-12-04T14:57:52,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/363ab5040b1946c8b2120bbfc0915d3b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/363ab5040b1946c8b2120bbfc0915d3b 2024-12-04T14:57:52,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/767f38054f434069863760065bc39944 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/767f38054f434069863760065bc39944 2024-12-04T14:57:52,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/ed5e6f88ef6344c792664ddd8e665526 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/ed5e6f88ef6344c792664ddd8e665526 2024-12-04T14:57:52,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/992edc3d01934a15a25811af9ef6e5af to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/992edc3d01934a15a25811af9ef6e5af 2024-12-04T14:57:52,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/05f13af59fd24191aa40eabe28b832f2 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/05f13af59fd24191aa40eabe28b832f2 2024-12-04T14:57:52,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0d27dccf06b2488fb7c462183b9b45fa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/7990148f04694410a6ac868ce03dc9bc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0420f103d2c64cc688b2a4610b2f2e97, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/fd9aaffc8c9a4eaea9c8de5fe4999d1a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/c89e3a9f06a448c1967756132856124b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2a07b499ef8748fcadb8aed421d23f0d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/5c6247971bbe4966883b40940aad1955, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/d332bbb49ed74b82ab5e3cfe91f7c9e5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/a9c4cf56dc39448e91a0865d0e092122, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/b8f3dede4854438f8803a3b6ccec508a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/bebb44ed8aff4201b949a9f4fc03fd1c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2f168624c41a4e249a839129aa515a47, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ba169fb190d841be81a0f2a44e075bc4] to archive 2024-12-04T14:57:52,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:57:52,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0d27dccf06b2488fb7c462183b9b45fa to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0d27dccf06b2488fb7c462183b9b45fa 2024-12-04T14:57:52,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/7990148f04694410a6ac868ce03dc9bc to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/7990148f04694410a6ac868ce03dc9bc 2024-12-04T14:57:53,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0420f103d2c64cc688b2a4610b2f2e97 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/0420f103d2c64cc688b2a4610b2f2e97 2024-12-04T14:57:53,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/fd9aaffc8c9a4eaea9c8de5fe4999d1a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/fd9aaffc8c9a4eaea9c8de5fe4999d1a 2024-12-04T14:57:53,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/c89e3a9f06a448c1967756132856124b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/c89e3a9f06a448c1967756132856124b 2024-12-04T14:57:53,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2a07b499ef8748fcadb8aed421d23f0d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2a07b499ef8748fcadb8aed421d23f0d 2024-12-04T14:57:53,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/5c6247971bbe4966883b40940aad1955 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/5c6247971bbe4966883b40940aad1955 2024-12-04T14:57:53,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/d332bbb49ed74b82ab5e3cfe91f7c9e5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/d332bbb49ed74b82ab5e3cfe91f7c9e5 2024-12-04T14:57:53,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/a9c4cf56dc39448e91a0865d0e092122 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/a9c4cf56dc39448e91a0865d0e092122 2024-12-04T14:57:53,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/b8f3dede4854438f8803a3b6ccec508a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/b8f3dede4854438f8803a3b6ccec508a 2024-12-04T14:57:53,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/bebb44ed8aff4201b949a9f4fc03fd1c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/bebb44ed8aff4201b949a9f4fc03fd1c 2024-12-04T14:57:53,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2f168624c41a4e249a839129aa515a47 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/2f168624c41a4e249a839129aa515a47 2024-12-04T14:57:53,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ba169fb190d841be81a0f2a44e075bc4 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ba169fb190d841be81a0f2a44e075bc4 2024-12-04T14:57:53,009 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/recovered.edits/260.seqid, newMaxSeqId=260, maxSeqId=4 2024-12-04T14:57:53,009 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. 2024-12-04T14:57:53,009 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 0ed254c309e31d6514a1b829d83e0676: Waiting for close lock at 1733324270469Running coprocessor pre-close hooks at 1733324270469Disabling compacts and flushes for region at 1733324270469Disabling writes for close at 1733324271305 (+836 ms)Obtaining lock to block concurrent updates at 1733324271305Preparing flush snapshotting stores in 0ed254c309e31d6514a1b829d83e0676 at 1733324271305Finished memstore snapshotting TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733324271306 (+1 ms)Flushing stores of TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676. at 1733324271306Flushing 0ed254c309e31d6514a1b829d83e0676/A: creating writer at 1733324271306Flushing mob file 0ed254c309e31d6514a1b829d83e0676/A: appending metadata at 1733324271310 (+4 ms)Flushing mob file 0ed254c309e31d6514a1b829d83e0676/A: closing flushed file at 1733324271310Flushing 0ed254c309e31d6514a1b829d83e0676/A: appending metadata at 1733324271718 (+408 ms)Flushing 0ed254c309e31d6514a1b829d83e0676/A: closing flushed file at 1733324271718Flushing 0ed254c309e31d6514a1b829d83e0676/B: creating writer at 1733324272126 (+408 ms)Flushing 0ed254c309e31d6514a1b829d83e0676/B: appending metadata at 1733324272128 (+2 ms)Flushing 0ed254c309e31d6514a1b829d83e0676/B: closing flushed file at 1733324272128Flushing 0ed254c309e31d6514a1b829d83e0676/C: creating writer at 1733324272537 (+409 ms)Flushing 0ed254c309e31d6514a1b829d83e0676/C: appending metadata at 1733324272540 (+3 ms)Flushing 0ed254c309e31d6514a1b829d83e0676/C: closing flushed file at 1733324272540Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65c8cf59: reopening flushed file at 1733324272954 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2014cfa5: reopening flushed file at 1733324272961 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@735d9b11: reopening flushed file at 1733324272966 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0ed254c309e31d6514a1b829d83e0676 in 1666ms, sequenceid=257, compaction requested=true at 1733324272971 (+5 ms)Writing region close event to WAL at 1733324273007 (+36 ms)Running coprocessor post-close hooks at 1733324273009 (+2 ms)Closed at 1733324273009 2024-12-04T14:57:53,011 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:53,011 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=0ed254c309e31d6514a1b829d83e0676, regionState=CLOSED 2024-12-04T14:57:53,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:57:53,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-04T14:57:53,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 0ed254c309e31d6514a1b829d83e0676, server=d214edae7bd2,34477,1733324158096 in 2.6960 sec 2024-12-04T14:57:53,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-04T14:57:53,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ed254c309e31d6514a1b829d83e0676, UNASSIGN in 2.7000 sec 2024-12-04T14:57:53,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-04T14:57:53,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.7030 sec 2024-12-04T14:57:53,019 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324273019"}]},"ts":"1733324273019"} 2024-12-04T14:57:53,020 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T14:57:53,021 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T14:57:53,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7140 sec 2024-12-04T14:57:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T14:57:54,475 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:54,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T14:57:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:54,477 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T14:57:54,478 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:54,481 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,485 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/recovered.edits] 2024-12-04T14:57:54,489 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/3940130b18e544c99926fa73fdb11322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/3940130b18e544c99926fa73fdb11322 2024-12-04T14:57:54,491 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/63f6e435bc9d4ed9b189c5e2461372d4 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/63f6e435bc9d4ed9b189c5e2461372d4 2024-12-04T14:57:54,493 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a76f5c98194b44e88f2bfc84db80dbfb to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/a76f5c98194b44e88f2bfc84db80dbfb 2024-12-04T14:57:54,496 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/ae0bcb0fe5294be1b4978860119635e3 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/ae0bcb0fe5294be1b4978860119635e3 2024-12-04T14:57:54,498 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/cdf5fd562c2f422eae8cfef1c784d5f4 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/A/cdf5fd562c2f422eae8cfef1c784d5f4 2024-12-04T14:57:54,501 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/4aa8257c725c4e6aa05f6dd93ce1bb1d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/4aa8257c725c4e6aa05f6dd93ce1bb1d 2024-12-04T14:57:54,503 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/9da45a995a13414380918fbf4e8e6d20 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/9da45a995a13414380918fbf4e8e6d20 2024-12-04T14:57:54,504 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/ac855a0fe3ef47e8b4f58af3a03f457c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/ac855a0fe3ef47e8b4f58af3a03f457c 2024-12-04T14:57:54,506 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/e35fc8bf424c44c0848610e542ec39a9 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/e35fc8bf424c44c0848610e542ec39a9 2024-12-04T14:57:54,508 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/f3831a52d2d84fdeb3fec5cd2fbc0d1b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/B/f3831a52d2d84fdeb3fec5cd2fbc0d1b 2024-12-04T14:57:54,510 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/1b6233bd72f14499bdccb3cbd548ffad to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/1b6233bd72f14499bdccb3cbd548ffad 2024-12-04T14:57:54,512 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/704dad97a6ac4b08b7236cac55f491e3 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/704dad97a6ac4b08b7236cac55f491e3 2024-12-04T14:57:54,513 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/c1fecc78cfe84aa08f1420731f2400fd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/c1fecc78cfe84aa08f1420731f2400fd 2024-12-04T14:57:54,514 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ddc33ca054854b0ba41b8908128c0e72 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ddc33ca054854b0ba41b8908128c0e72 2024-12-04T14:57:54,516 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ebd91f443ae64e8f88af795f4c5ab259 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/C/ebd91f443ae64e8f88af795f4c5ab259 2024-12-04T14:57:54,518 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/recovered.edits/260.seqid to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676/recovered.edits/260.seqid 2024-12-04T14:57:54,519 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,519 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T14:57:54,519 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T14:57:54,520 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T14:57:54,522 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040f3e12440f824dc8a582055a8800201b_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040f3e12440f824dc8a582055a8800201b_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,523 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041e61fce87c564e58abd8815399f7d21f_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041e61fce87c564e58abd8815399f7d21f_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,524 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120428b86834852d44b5a9669e0460aba9ab_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120428b86834852d44b5a9669e0460aba9ab_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,525 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042a0a442d0fee4837a80be4f1d04afee3_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042a0a442d0fee4837a80be4f1d04afee3_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,526 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465652ef3156f4613ac3f031c7f1b8492_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465652ef3156f4613ac3f031c7f1b8492_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,526 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047a250536397a4784ad3ce6b9466c4097_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047a250536397a4784ad3ce6b9466c4097_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,527 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048afa33e8fc474eb1b6a69d1148a29db8_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048afa33e8fc474eb1b6a69d1148a29db8_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,528 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120498a229a6c4bd421dabb4c8768523d664_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120498a229a6c4bd421dabb4c8768523d664_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,528 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a204242c34524abdba26320bcd07b2fe_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a204242c34524abdba26320bcd07b2fe_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,529 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c666942a191b40e2813270eea39b8847_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c666942a191b40e2813270eea39b8847_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,530 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d5c312d115cb47c7a67c8ee137a91130_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d5c312d115cb47c7a67c8ee137a91130_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,531 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e19d2365109d4d4f8c784580474ece34_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e19d2365109d4d4f8c784580474ece34_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,531 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e266177aedee428a94259ac1e45af0f9_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e266177aedee428a94259ac1e45af0f9_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,532 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204fec87a700e654d749260e775d4b558ee_0ed254c309e31d6514a1b829d83e0676 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204fec87a700e654d749260e775d4b558ee_0ed254c309e31d6514a1b829d83e0676 2024-12-04T14:57:54,532 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T14:57:54,534 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:54,536 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T14:57:54,537 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T14:57:54,538 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:54,538 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T14:57:54,538 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324274538"}]},"ts":"9223372036854775807"} 2024-12-04T14:57:54,540 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T14:57:54,540 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0ed254c309e31d6514a1b829d83e0676, NAME => 'TestAcidGuarantees,,1733324246140.0ed254c309e31d6514a1b829d83e0676.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T14:57:54,540 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T14:57:54,540 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324274540"}]},"ts":"9223372036854775807"} 2024-12-04T14:57:54,541 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T14:57:54,542 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:54,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-12-04T14:57:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T14:57:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T14:57:54,585 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:54,595 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 239) - Thread LEAK? -, OpenFileDescriptor=463 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=399 (was 348) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9602 (was 8713) - AvailableMemoryMB LEAK? - 2024-12-04T14:57:54,603 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=399, ProcessCount=11, AvailableMemoryMB=9602 2024-12-04T14:57:54,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:57:54,605 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:57:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T14:57:54,606 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:57:54,607 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:54,607 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-12-04T14:57:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T14:57:54,608 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:57:54,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742202_1378 (size=959) 2024-12-04T14:57:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T14:57:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T14:57:55,015 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1 2024-12-04T14:57:55,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742203_1379 (size=53) 2024-12-04T14:57:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T14:57:55,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:57:55,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 031e210d034f5b12ccb5e29a336d9de0, disabling compactions & flushes 2024-12-04T14:57:55,420 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:55,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:55,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. after waiting 0 ms 2024-12-04T14:57:55,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:55,420 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:55,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 031e210d034f5b12ccb5e29a336d9de0: Waiting for close lock at 1733324275420Disabling compacts and flushes for region at 1733324275420Disabling writes for close at 1733324275420Writing region close event to WAL at 1733324275420Closed at 1733324275420 2024-12-04T14:57:55,422 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:57:55,422 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324275422"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324275422"}]},"ts":"1733324275422"} 2024-12-04T14:57:55,423 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T14:57:55,424 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:57:55,424 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324275424"}]},"ts":"1733324275424"} 2024-12-04T14:57:55,425 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T14:57:55,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=031e210d034f5b12ccb5e29a336d9de0, ASSIGN}] 2024-12-04T14:57:55,426 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=031e210d034f5b12ccb5e29a336d9de0, ASSIGN 2024-12-04T14:57:55,427 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=031e210d034f5b12ccb5e29a336d9de0, ASSIGN; state=OFFLINE, location=d214edae7bd2,34477,1733324158096; forceNewPlan=false, retain=false 2024-12-04T14:57:55,578 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=031e210d034f5b12ccb5e29a336d9de0, regionState=OPENING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:55,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=031e210d034f5b12ccb5e29a336d9de0, ASSIGN because future has completed 2024-12-04T14:57:55,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:57:55,737 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:55,737 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:57:55,739 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,739 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:57:55,739 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,739 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,742 INFO [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,745 INFO [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 031e210d034f5b12ccb5e29a336d9de0 columnFamilyName A 2024-12-04T14:57:55,745 DEBUG [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T14:57:55,746 INFO [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] regionserver.HStore(327): Store=031e210d034f5b12ccb5e29a336d9de0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:57:55,746 INFO [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,747 INFO [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 031e210d034f5b12ccb5e29a336d9de0 columnFamilyName B 2024-12-04T14:57:55,747 DEBUG [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:55,748 INFO [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] regionserver.HStore(327): Store=031e210d034f5b12ccb5e29a336d9de0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:57:55,748 INFO [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,749 INFO [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 031e210d034f5b12ccb5e29a336d9de0 columnFamilyName C 2024-12-04T14:57:55,749 DEBUG [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:57:55,750 INFO [StoreOpener-031e210d034f5b12ccb5e29a336d9de0-1 {}] regionserver.HStore(327): Store=031e210d034f5b12ccb5e29a336d9de0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:57:55,750 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,751 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,751 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,752 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,752 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,753 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:57:55,754 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,756 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:57:55,757 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened 031e210d034f5b12ccb5e29a336d9de0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65606825, jitterRate=-0.02238212525844574}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:57:55,757 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:55,757 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for 031e210d034f5b12ccb5e29a336d9de0: Running coprocessor pre-open hook at 1733324275739Writing region info on filesystem at 1733324275740 (+1 ms)Initializing all the Stores at 1733324275741 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324275741Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324275742 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324275742Cleaning up temporary data from old regions at 1733324275752 (+10 ms)Running coprocessor post-open hooks at 1733324275757 (+5 ms)Region opened successfully at 1733324275757 2024-12-04T14:57:55,758 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., pid=112, masterSystemTime=1733324275731 2024-12-04T14:57:55,760 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:55,760 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:55,761 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=031e210d034f5b12ccb5e29a336d9de0, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:55,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:57:55,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-04T14:57:55,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure 031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 in 184 msec 2024-12-04T14:57:55,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-04T14:57:55,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=031e210d034f5b12ccb5e29a336d9de0, ASSIGN in 341 msec 2024-12-04T14:57:55,769 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:57:55,769 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324275769"}]},"ts":"1733324275769"} 2024-12-04T14:57:55,770 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T14:57:55,771 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:57:55,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-12-04T14:57:56,577 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:57:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T14:57:56,755 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:56,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,757 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19614cbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,759 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@649a925d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,760 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,762 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:57:56,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,45797,1733324157426 2024-12-04T14:57:56,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@72880ad4 2024-12-04T14:57:56,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:57:56,764 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:57:56,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,766 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4977099d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541df7f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,769 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fef277a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,770 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8ac720, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,772 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4503c6a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,773 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158bacee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,775 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472fd16a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,776 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62e029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,778 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@215ecc88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,779 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2280981b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,784 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a04b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,785 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794ee972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,787 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@640d9dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,788 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ac7eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,791 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfd8b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0a9bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,794 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73740007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,796 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@717c214b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:57:56,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:57:56,798 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:57:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:57:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:57:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3bf6ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:57:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:57:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:57:56,799 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:57:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f2f66c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:57:56,801 DEBUG [Thread-1746 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,801 DEBUG [Thread-1731 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,801 DEBUG [Thread-1735 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,802 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,802 DEBUG [Thread-1742 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,802 DEBUG [Thread-1733 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-04T14:57:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,803 DEBUG [Thread-1748 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,803 DEBUG [Thread-1750 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,805 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T14:57:56,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:57:56,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:57:56,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:56,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:56,808 DEBUG [Thread-1737 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:57:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:57:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:57:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:57:56,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:57:56,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,815 DEBUG [Thread-1739 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:57:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:56,820 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:57:56,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:57:56,821 DEBUG [Thread-1744 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:57:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:57:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:57:56,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:57:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2] 2024-12-04T14:57:56,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/63c8e2686dec41c2983ef6c3f7b934b1 is 50, key is test_row_0/A:col10/1733324276819/Put/seqid=0 2024-12-04T14:57:56,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742204_1380 (size=14341) 2024-12-04T14:57:56,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/63c8e2686dec41c2983ef6c3f7b934b1 2024-12-04T14:57:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324336868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:56,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324336869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324336870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:56,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/c997abbab56c4f93847d9fdb1edba618 is 50, key is test_row_0/B:col10/1733324276819/Put/seqid=0 2024-12-04T14:57:56,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324336879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:56,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324336881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:56,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:56,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742205_1381 (size=12001) 2024-12-04T14:57:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T14:57:56,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T14:57:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:56,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:56,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324336985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324336985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:56,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:56,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324336985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324337011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324337015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T14:57:57,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:57,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T14:57:57,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324337195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324337195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324337205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324337224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324337224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T14:57:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/c997abbab56c4f93847d9fdb1edba618 2024-12-04T14:57:57,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/732e0482a3ed40d0943bfee816cb2f17 is 50, key is test_row_0/C:col10/1733324276819/Put/seqid=0 2024-12-04T14:57:57,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742206_1382 (size=12001) 2024-12-04T14:57:57,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T14:57:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T14:57:57,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324337505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324337505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324337515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324337535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324337535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:57,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T14:57:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T14:57:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:57,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:57,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/732e0482a3ed40d0943bfee816cb2f17 2024-12-04T14:57:57,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/63c8e2686dec41c2983ef6c3f7b934b1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/63c8e2686dec41c2983ef6c3f7b934b1 2024-12-04T14:57:57,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/63c8e2686dec41c2983ef6c3f7b934b1, entries=200, sequenceid=13, filesize=14.0 K 2024-12-04T14:57:57,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/c997abbab56c4f93847d9fdb1edba618 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/c997abbab56c4f93847d9fdb1edba618 2024-12-04T14:57:57,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/c997abbab56c4f93847d9fdb1edba618, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T14:57:57,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/732e0482a3ed40d0943bfee816cb2f17 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/732e0482a3ed40d0943bfee816cb2f17 2024-12-04T14:57:57,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/732e0482a3ed40d0943bfee816cb2f17, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T14:57:57,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 031e210d034f5b12ccb5e29a336d9de0 in 939ms, sequenceid=13, compaction requested=false 2024-12-04T14:57:57,759 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T14:57:57,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:57:57,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-04T14:57:57,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:57,881 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:57:57,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/fb5f9733fc1e47e7b33aa1bd9d8176a1 is 50, key is test_row_0/A:col10/1733324276878/Put/seqid=0 2024-12-04T14:57:57,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742207_1383 (size=12001) 2024-12-04T14:57:57,901 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/fb5f9733fc1e47e7b33aa1bd9d8176a1 2024-12-04T14:57:57,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/5af0224f2dd74578bb53d62323b1417f is 50, key is test_row_0/B:col10/1733324276878/Put/seqid=0 2024-12-04T14:57:57,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742208_1384 (size=12001) 2024-12-04T14:57:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T14:57:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:58,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:58,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324338045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324338045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324338059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324338059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324338059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324338175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324338175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324338175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,311 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/5af0224f2dd74578bb53d62323b1417f 2024-12-04T14:57:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/1de1a0d3c08546feaca749642a9eb9d5 is 50, key is test_row_0/C:col10/1733324276878/Put/seqid=0 2024-12-04T14:57:58,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742209_1385 (size=12001) 2024-12-04T14:57:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324338385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324338385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324338385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,526 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:57:58,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324338694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324338694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324338705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:58,726 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/1de1a0d3c08546feaca749642a9eb9d5 2024-12-04T14:57:58,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/fb5f9733fc1e47e7b33aa1bd9d8176a1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/fb5f9733fc1e47e7b33aa1bd9d8176a1 2024-12-04T14:57:58,733 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/fb5f9733fc1e47e7b33aa1bd9d8176a1, entries=150, sequenceid=38, filesize=11.7 K 2024-12-04T14:57:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/5af0224f2dd74578bb53d62323b1417f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5af0224f2dd74578bb53d62323b1417f 2024-12-04T14:57:58,736 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5af0224f2dd74578bb53d62323b1417f, entries=150, sequenceid=38, filesize=11.7 K 2024-12-04T14:57:58,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/1de1a0d3c08546feaca749642a9eb9d5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1de1a0d3c08546feaca749642a9eb9d5 2024-12-04T14:57:58,740 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1de1a0d3c08546feaca749642a9eb9d5, entries=150, sequenceid=38, filesize=11.7 K 2024-12-04T14:57:58,740 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 031e210d034f5b12ccb5e29a336d9de0 in 859ms, sequenceid=38, compaction requested=false 2024-12-04T14:57:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:57:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-04T14:57:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-04T14:57:58,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-04T14:57:58,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9350 sec 2024-12-04T14:57:58,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.9420 sec 2024-12-04T14:57:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-04T14:57:58,955 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:57:58,955 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:57:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-04T14:57:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T14:57:58,957 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:57:58,957 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:57:58,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:57:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T14:57:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:57:59,078 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:57:59,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-04T14:57:59,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/73740ed1833f4cc09a7c3ae5644e776f is 50, key is test_row_0/A:col10/1733324279077/Put/seqid=0 2024-12-04T14:57:59,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742210_1386 (size=14341) 2024-12-04T14:57:59,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T14:57:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324339144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324339145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324339205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324339205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324339214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324339255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T14:57:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324339265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T14:57:59,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T14:57:59,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324339465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324339475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/73740ed1833f4cc09a7c3ae5644e776f 2024-12-04T14:57:59,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/ad9b827d720f4657bd884f4bae91c50a is 50, key is test_row_0/B:col10/1733324279077/Put/seqid=0 2024-12-04T14:57:59,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742211_1387 (size=12001) 2024-12-04T14:57:59,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T14:57:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T14:57:59,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T14:57:59,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:59,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324339775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:57:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324339794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:57:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:57:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:57:59,871 WARN [regionserver/d214edae7bd2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 4, running: 1 2024-12-04T14:57:59,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T14:57:59,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:57:59,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:57:59,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/ad9b827d720f4657bd884f4bae91c50a 2024-12-04T14:57:59,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/965dc71061e84982bcde94c00afc8f9b is 50, key is test_row_0/C:col10/1733324279077/Put/seqid=0 2024-12-04T14:57:59,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742212_1388 (size=12001) 2024-12-04T14:57:59,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/965dc71061e84982bcde94c00afc8f9b 2024-12-04T14:57:59,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/73740ed1833f4cc09a7c3ae5644e776f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/73740ed1833f4cc09a7c3ae5644e776f 2024-12-04T14:57:59,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/73740ed1833f4cc09a7c3ae5644e776f, entries=200, sequenceid=51, filesize=14.0 K 2024-12-04T14:57:59,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/ad9b827d720f4657bd884f4bae91c50a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/ad9b827d720f4657bd884f4bae91c50a 2024-12-04T14:57:59,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/ad9b827d720f4657bd884f4bae91c50a, entries=150, sequenceid=51, filesize=11.7 K 2024-12-04T14:57:59,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/965dc71061e84982bcde94c00afc8f9b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/965dc71061e84982bcde94c00afc8f9b 2024-12-04T14:57:59,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/965dc71061e84982bcde94c00afc8f9b, entries=150, sequenceid=51, filesize=11.7 K 2024-12-04T14:57:59,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 031e210d034f5b12ccb5e29a336d9de0 in 856ms, sequenceid=51, compaction requested=true 2024-12-04T14:57:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:57:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:57:59,934 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:59,934 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:57:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:57:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:59,947 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:59,947 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/A is initiating minor compaction (all files) 2024-12-04T14:57:59,947 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/A in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,947 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/63c8e2686dec41c2983ef6c3f7b934b1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/fb5f9733fc1e47e7b33aa1bd9d8176a1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/73740ed1833f4cc09a7c3ae5644e776f] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=39.7 K 2024-12-04T14:57:59,947 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:59,947 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/B is initiating minor compaction (all files) 2024-12-04T14:57:59,947 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/B in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,947 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/c997abbab56c4f93847d9fdb1edba618, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5af0224f2dd74578bb53d62323b1417f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/ad9b827d720f4657bd884f4bae91c50a] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=35.2 K 2024-12-04T14:57:59,948 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63c8e2686dec41c2983ef6c3f7b934b1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733324276816 2024-12-04T14:57:59,948 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c997abbab56c4f93847d9fdb1edba618, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733324276818 2024-12-04T14:57:59,948 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb5f9733fc1e47e7b33aa1bd9d8176a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733324276869 2024-12-04T14:57:59,948 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 5af0224f2dd74578bb53d62323b1417f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733324276869 2024-12-04T14:57:59,949 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73740ed1833f4cc09a7c3ae5644e776f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733324278025 2024-12-04T14:57:59,949 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting ad9b827d720f4657bd884f4bae91c50a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733324278025 2024-12-04T14:57:59,958 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#A#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:59,959 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/185f7bb1fdb94f3e911487efef0ca85d is 50, key is test_row_0/A:col10/1733324279077/Put/seqid=0 2024-12-04T14:57:59,959 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#B#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:59,960 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/305ba369f4124e098c505dd53429aa68 is 50, key is test_row_0/B:col10/1733324279077/Put/seqid=0 2024-12-04T14:57:59,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742214_1390 (size=12104) 2024-12-04T14:57:59,978 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/185f7bb1fdb94f3e911487efef0ca85d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/185f7bb1fdb94f3e911487efef0ca85d 2024-12-04T14:57:59,982 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/A of 031e210d034f5b12ccb5e29a336d9de0 into 185f7bb1fdb94f3e911487efef0ca85d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:59,982 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:57:59,982 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/A, priority=13, startTime=1733324279934; duration=0sec 2024-12-04T14:57:59,982 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:57:59,982 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:A 2024-12-04T14:57:59,982 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:57:59,983 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:57:59,983 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/C is initiating minor compaction (all files) 2024-12-04T14:57:59,983 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/C in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:57:59,983 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/732e0482a3ed40d0943bfee816cb2f17, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1de1a0d3c08546feaca749642a9eb9d5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/965dc71061e84982bcde94c00afc8f9b] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=35.2 K 2024-12-04T14:57:59,984 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 732e0482a3ed40d0943bfee816cb2f17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733324276818 2024-12-04T14:57:59,984 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1de1a0d3c08546feaca749642a9eb9d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733324276869 2024-12-04T14:57:59,986 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 965dc71061e84982bcde94c00afc8f9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733324278025 2024-12-04T14:57:59,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742213_1389 (size=12104) 2024-12-04T14:57:59,992 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/305ba369f4124e098c505dd53429aa68 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/305ba369f4124e098c505dd53429aa68 2024-12-04T14:57:59,992 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:57:59,992 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/21e740a42aaa4b04969f0250f4601eaa is 50, key is test_row_0/C:col10/1733324279077/Put/seqid=0 2024-12-04T14:57:59,995 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/B of 031e210d034f5b12ccb5e29a336d9de0 into 305ba369f4124e098c505dd53429aa68(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:57:59,995 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:57:59,995 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/B, priority=13, startTime=1733324279934; duration=0sec 2024-12-04T14:57:59,995 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:57:59,996 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:B 2024-12-04T14:57:59,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742215_1391 (size=12104) 2024-12-04T14:58:00,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T14:58:00,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:00,027 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-04T14:58:00,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/3eb6ad233cfa40faa2b797cfe5c25db6 is 50, key is test_row_0/A:col10/1733324279137/Put/seqid=0 2024-12-04T14:58:00,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742216_1392 (size=12001) 2024-12-04T14:58:00,035 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/3eb6ad233cfa40faa2b797cfe5c25db6 2024-12-04T14:58:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/94b8bed6fcd7420f9c93228118ede395 is 50, key is test_row_0/B:col10/1733324279137/Put/seqid=0 2024-12-04T14:58:00,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742217_1393 (size=12001) 2024-12-04T14:58:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T14:58:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:00,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:00,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324340242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324340247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324340248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324340285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324340305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324340354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324340355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324340355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,404 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/21e740a42aaa4b04969f0250f4601eaa as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/21e740a42aaa4b04969f0250f4601eaa 2024-12-04T14:58:00,411 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/C of 031e210d034f5b12ccb5e29a336d9de0 into 21e740a42aaa4b04969f0250f4601eaa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:00,411 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:00,411 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/C, priority=13, startTime=1733324279934; duration=0sec 2024-12-04T14:58:00,411 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:00,411 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:C 2024-12-04T14:58:00,445 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/94b8bed6fcd7420f9c93228118ede395 2024-12-04T14:58:00,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/582d643f367b44e885832ba2fd196cec is 50, key is test_row_0/C:col10/1733324279137/Put/seqid=0 2024-12-04T14:58:00,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742218_1394 (size=12001) 2024-12-04T14:58:00,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324340565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324340565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324340565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,856 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/582d643f367b44e885832ba2fd196cec 2024-12-04T14:58:00,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/3eb6ad233cfa40faa2b797cfe5c25db6 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3eb6ad233cfa40faa2b797cfe5c25db6 2024-12-04T14:58:00,864 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3eb6ad233cfa40faa2b797cfe5c25db6, entries=150, sequenceid=76, filesize=11.7 K 2024-12-04T14:58:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/94b8bed6fcd7420f9c93228118ede395 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/94b8bed6fcd7420f9c93228118ede395 2024-12-04T14:58:00,869 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/94b8bed6fcd7420f9c93228118ede395, entries=150, sequenceid=76, filesize=11.7 K 2024-12-04T14:58:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/582d643f367b44e885832ba2fd196cec as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/582d643f367b44e885832ba2fd196cec 2024-12-04T14:58:00,873 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/582d643f367b44e885832ba2fd196cec, entries=150, sequenceid=76, filesize=11.7 K 2024-12-04T14:58:00,874 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 031e210d034f5b12ccb5e29a336d9de0 in 847ms, sequenceid=76, compaction requested=false 2024-12-04T14:58:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-04T14:58:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-04T14:58:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:00,879 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:00,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-04T14:58:00,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-04T14:58:00,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9200 sec 2024-12-04T14:58:00,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.9260 sec 2024-12-04T14:58:00,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/094a60d3e78c48fbb4a07456d16aac0e is 50, key is test_row_0/A:col10/1733324280246/Put/seqid=0 2024-12-04T14:58:00,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742219_1395 (size=12001) 2024-12-04T14:58:00,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324340925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324340926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324340931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324341035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324341035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324341035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T14:58:01,105 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:01,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-04T14:58:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T14:58:01,107 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:01,107 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:01,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T14:58:01,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324341245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324341245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324341245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T14:58:01,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:01,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/094a60d3e78c48fbb4a07456d16aac0e 2024-12-04T14:58:01,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/198f8b0c989a4558a089d11d60b624bd is 50, key is test_row_0/B:col10/1733324280246/Put/seqid=0 2024-12-04T14:58:01,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742220_1396 (size=12001) 2024-12-04T14:58:01,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324341295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,301 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 031e210d034f5b12ccb5e29a336d9de0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324341325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,329 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 031e210d034f5b12ccb5e29a336d9de0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T14:58:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T14:58:01,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324341555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324341555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T14:58:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324341565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:01,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/198f8b0c989a4558a089d11d60b624bd 2024-12-04T14:58:01,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T14:58:01,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:01,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:01,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/29278334d4ef471083a3a0098712cdda is 50, key is test_row_0/C:col10/1733324280246/Put/seqid=0 2024-12-04T14:58:01,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742221_1397 (size=12001) 2024-12-04T14:58:01,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/29278334d4ef471083a3a0098712cdda 2024-12-04T14:58:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T14:58:01,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/094a60d3e78c48fbb4a07456d16aac0e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/094a60d3e78c48fbb4a07456d16aac0e 2024-12-04T14:58:01,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/094a60d3e78c48fbb4a07456d16aac0e, entries=150, sequenceid=91, filesize=11.7 K 2024-12-04T14:58:01,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/198f8b0c989a4558a089d11d60b624bd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/198f8b0c989a4558a089d11d60b624bd 2024-12-04T14:58:01,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/198f8b0c989a4558a089d11d60b624bd, entries=150, sequenceid=91, filesize=11.7 K 2024-12-04T14:58:01,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/29278334d4ef471083a3a0098712cdda as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/29278334d4ef471083a3a0098712cdda 2024-12-04T14:58:01,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/29278334d4ef471083a3a0098712cdda, entries=150, sequenceid=91, filesize=11.7 K 2024-12-04T14:58:01,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 031e210d034f5b12ccb5e29a336d9de0 in 877ms, sequenceid=91, compaction requested=true 2024-12-04T14:58:01,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:01,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:01,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:01,756 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:01,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:01,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:01,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:01,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T14:58:01,757 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:01,757 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/A is initiating minor compaction (all files) 2024-12-04T14:58:01,757 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/A in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,757 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/185f7bb1fdb94f3e911487efef0ca85d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3eb6ad233cfa40faa2b797cfe5c25db6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/094a60d3e78c48fbb4a07456d16aac0e] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=35.3 K 2024-12-04T14:58:01,758 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185f7bb1fdb94f3e911487efef0ca85d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733324278025 2024-12-04T14:58:01,758 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb6ad233cfa40faa2b797cfe5c25db6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324279137 2024-12-04T14:58:01,758 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:01,759 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 094a60d3e78c48fbb4a07456d16aac0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733324280239 2024-12-04T14:58:01,759 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:01,759 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/B is initiating minor compaction (all files) 2024-12-04T14:58:01,759 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/B in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,759 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/305ba369f4124e098c505dd53429aa68, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/94b8bed6fcd7420f9c93228118ede395, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/198f8b0c989a4558a089d11d60b624bd] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=35.3 K 2024-12-04T14:58:01,759 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 305ba369f4124e098c505dd53429aa68, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733324278025 2024-12-04T14:58:01,759 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b8bed6fcd7420f9c93228118ede395, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324279137 2024-12-04T14:58:01,764 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#A#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:01,764 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/36df617fa6cf4fa3a2a19894435b0df6 is 50, key is test_row_0/A:col10/1733324280246/Put/seqid=0 2024-12-04T14:58:01,770 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 198f8b0c989a4558a089d11d60b624bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733324280239 2024-12-04T14:58:01,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742222_1398 (size=12207) 2024-12-04T14:58:01,780 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:01,781 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/f2d449f814be46a49cd01f765be5fbf5 is 50, key is test_row_0/B:col10/1733324280246/Put/seqid=0 2024-12-04T14:58:01,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742223_1399 (size=12207) 2024-12-04T14:58:01,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T14:58:01,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:01,870 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-04T14:58:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/bb88141640124429867e4114ae48a83b is 50, key is test_row_0/A:col10/1733324280924/Put/seqid=0 2024-12-04T14:58:01,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742224_1400 (size=12001) 2024-12-04T14:58:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:02,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:02,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324342088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324342092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324342094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,176 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/36df617fa6cf4fa3a2a19894435b0df6 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/36df617fa6cf4fa3a2a19894435b0df6 2024-12-04T14:58:02,180 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/A of 031e210d034f5b12ccb5e29a336d9de0 into 36df617fa6cf4fa3a2a19894435b0df6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:02,180 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:02,180 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/A, priority=13, startTime=1733324281756; duration=0sec 2024-12-04T14:58:02,180 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:02,180 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:A 2024-12-04T14:58:02,180 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:02,181 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:02,181 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/C is initiating minor compaction (all files) 2024-12-04T14:58:02,181 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/C in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:02,181 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/21e740a42aaa4b04969f0250f4601eaa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/582d643f367b44e885832ba2fd196cec, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/29278334d4ef471083a3a0098712cdda] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=35.3 K 2024-12-04T14:58:02,181 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e740a42aaa4b04969f0250f4601eaa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733324278025 2024-12-04T14:58:02,181 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 582d643f367b44e885832ba2fd196cec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733324279137 2024-12-04T14:58:02,181 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29278334d4ef471083a3a0098712cdda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733324280239 2024-12-04T14:58:02,187 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:02,187 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/2d18c426621140dea250ad17177f6bda is 50, key is test_row_0/C:col10/1733324280246/Put/seqid=0 2024-12-04T14:58:02,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742225_1401 (size=12207) 2024-12-04T14:58:02,195 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/f2d449f814be46a49cd01f765be5fbf5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f2d449f814be46a49cd01f765be5fbf5 2024-12-04T14:58:02,198 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/B of 031e210d034f5b12ccb5e29a336d9de0 into f2d449f814be46a49cd01f765be5fbf5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:02,198 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:02,198 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/B, priority=13, startTime=1733324281756; duration=0sec 2024-12-04T14:58:02,198 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:02,198 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:B 2024-12-04T14:58:02,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324342205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324342205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324342205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T14:58:02,278 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/bb88141640124429867e4114ae48a83b 2024-12-04T14:58:02,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/06de4e12eac1414c98537a884cbab90a is 50, key is test_row_0/B:col10/1733324280924/Put/seqid=0 2024-12-04T14:58:02,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742226_1402 (size=12001) 2024-12-04T14:58:02,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324342425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324342425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324342425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,597 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/2d18c426621140dea250ad17177f6bda as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/2d18c426621140dea250ad17177f6bda 2024-12-04T14:58:02,603 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/C of 031e210d034f5b12ccb5e29a336d9de0 into 2d18c426621140dea250ad17177f6bda(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:02,603 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:02,603 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/C, priority=13, startTime=1733324281756; duration=0sec 2024-12-04T14:58:02,603 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:02,603 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:C 2024-12-04T14:58:02,687 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/06de4e12eac1414c98537a884cbab90a 2024-12-04T14:58:02,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/7dded204e9f94d3e80ae07bc772f4864 is 50, key is test_row_0/C:col10/1733324280924/Put/seqid=0 2024-12-04T14:58:02,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742227_1403 (size=12001) 2024-12-04T14:58:02,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324342735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324342735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:02,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324342735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,109 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/7dded204e9f94d3e80ae07bc772f4864 2024-12-04T14:58:03,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/bb88141640124429867e4114ae48a83b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/bb88141640124429867e4114ae48a83b 2024-12-04T14:58:03,129 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/bb88141640124429867e4114ae48a83b, entries=150, sequenceid=113, filesize=11.7 K 2024-12-04T14:58:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/06de4e12eac1414c98537a884cbab90a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/06de4e12eac1414c98537a884cbab90a 2024-12-04T14:58:03,134 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/06de4e12eac1414c98537a884cbab90a, entries=150, sequenceid=113, filesize=11.7 K 2024-12-04T14:58:03,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/7dded204e9f94d3e80ae07bc772f4864 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/7dded204e9f94d3e80ae07bc772f4864 2024-12-04T14:58:03,160 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/7dded204e9f94d3e80ae07bc772f4864, entries=150, sequenceid=113, filesize=11.7 K 2024-12-04T14:58:03,166 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 031e210d034f5b12ccb5e29a336d9de0 in 1296ms, sequenceid=113, compaction requested=false 2024-12-04T14:58:03,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:03,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-04T14:58:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-04T14:58:03,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-04T14:58:03,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0600 sec 2024-12-04T14:58:03,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.0670 sec 2024-12-04T14:58:03,252 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:03,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-04T14:58:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T14:58:03,255 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:03,256 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-04T14:58:03,258 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:03,260 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:03,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T14:58:03,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/4614de8e33e14b14baba09f5bb516df2 is 50, key is test_row_0/A:col10/1733324282087/Put/seqid=0 2024-12-04T14:58:03,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742228_1404 (size=12101) 2024-12-04T14:58:03,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/4614de8e33e14b14baba09f5bb516df2 2024-12-04T14:58:03,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/cbbe849aeb9940faa105d33460b5f14f is 50, key is test_row_0/B:col10/1733324282087/Put/seqid=0 2024-12-04T14:58:03,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324343306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,322 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 031e210d034f5b12ccb5e29a336d9de0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:03,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324343316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742229_1405 (size=12101) 2024-12-04T14:58:03,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/cbbe849aeb9940faa105d33460b5f14f 2024-12-04T14:58:03,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324343320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324343322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/fcec02e77d2f4814b425a61f802a2572 is 50, key is test_row_0/C:col10/1733324282087/Put/seqid=0 2024-12-04T14:58:03,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324343355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,361 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 031e210d034f5b12ccb5e29a336d9de0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T14:58:03,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742230_1406 (size=12101) 2024-12-04T14:58:03,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T14:58:03,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:03,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:03,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:03,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324343435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324343445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324343445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T14:58:03,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:03,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:03,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T14:58:03,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324343645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324343655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324343655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:03,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:03,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T14:58:03,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:03,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:03,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:03,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/fcec02e77d2f4814b425a61f802a2572 2024-12-04T14:58:03,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/4614de8e33e14b14baba09f5bb516df2 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4614de8e33e14b14baba09f5bb516df2 2024-12-04T14:58:03,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4614de8e33e14b14baba09f5bb516df2, entries=150, sequenceid=133, filesize=11.8 K 2024-12-04T14:58:03,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/cbbe849aeb9940faa105d33460b5f14f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/cbbe849aeb9940faa105d33460b5f14f 2024-12-04T14:58:03,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/cbbe849aeb9940faa105d33460b5f14f, entries=150, sequenceid=133, filesize=11.8 K 2024-12-04T14:58:03,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/fcec02e77d2f4814b425a61f802a2572 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/fcec02e77d2f4814b425a61f802a2572 2024-12-04T14:58:03,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/fcec02e77d2f4814b425a61f802a2572, entries=150, sequenceid=133, filesize=11.8 K 2024-12-04T14:58:03,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 031e210d034f5b12ccb5e29a336d9de0 in 577ms, sequenceid=133, compaction requested=true 2024-12-04T14:58:03,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:03,831 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:03,832 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:03,833 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:03,833 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/A is initiating minor compaction (all files) 2024-12-04T14:58:03,833 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/A in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,833 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/36df617fa6cf4fa3a2a19894435b0df6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/bb88141640124429867e4114ae48a83b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4614de8e33e14b14baba09f5bb516df2] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=35.5 K 2024-12-04T14:58:03,833 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:03,833 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/B is initiating minor compaction (all files) 2024-12-04T14:58:03,833 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/B in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,833 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f2d449f814be46a49cd01f765be5fbf5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/06de4e12eac1414c98537a884cbab90a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/cbbe849aeb9940faa105d33460b5f14f] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=35.5 K 2024-12-04T14:58:03,833 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36df617fa6cf4fa3a2a19894435b0df6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733324280239 2024-12-04T14:58:03,834 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d449f814be46a49cd01f765be5fbf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733324280239 2024-12-04T14:58:03,834 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb88141640124429867e4114ae48a83b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733324280922 2024-12-04T14:58:03,834 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 06de4e12eac1414c98537a884cbab90a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733324280922 2024-12-04T14:58:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:03,834 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4614de8e33e14b14baba09f5bb516df2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733324282087 2024-12-04T14:58:03,834 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting cbbe849aeb9940faa105d33460b5f14f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733324282087 2024-12-04T14:58:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:03,865 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:03,866 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/e8dd08d212f740aab46fee10f7f62ee7 is 50, key is test_row_0/B:col10/1733324282087/Put/seqid=0 2024-12-04T14:58:03,873 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#A#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:03,874 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/939d04133a7a40d79a8ec19cd3b49dcc is 50, key is test_row_0/A:col10/1733324282087/Put/seqid=0 2024-12-04T14:58:03,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T14:58:03,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,879 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-04T14:58:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T14:58:03,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/c60fb86da5f94e0385cfcae98a07ceda is 50, key is test_row_0/A:col10/1733324283320/Put/seqid=0 2024-12-04T14:58:03,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742231_1407 (size=12409) 2024-12-04T14:58:03,930 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/e8dd08d212f740aab46fee10f7f62ee7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e8dd08d212f740aab46fee10f7f62ee7 2024-12-04T14:58:03,937 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/B of 031e210d034f5b12ccb5e29a336d9de0 into e8dd08d212f740aab46fee10f7f62ee7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:03,937 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:03,937 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/B, priority=13, startTime=1733324283832; duration=0sec 2024-12-04T14:58:03,938 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:03,938 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:B 2024-12-04T14:58:03,938 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:03,938 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:03,939 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/C is initiating minor compaction (all files) 2024-12-04T14:58:03,939 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/C in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:03,939 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/2d18c426621140dea250ad17177f6bda, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/7dded204e9f94d3e80ae07bc772f4864, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/fcec02e77d2f4814b425a61f802a2572] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=35.5 K 2024-12-04T14:58:03,939 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d18c426621140dea250ad17177f6bda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733324280239 2024-12-04T14:58:03,939 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dded204e9f94d3e80ae07bc772f4864, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733324280922 2024-12-04T14:58:03,939 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting fcec02e77d2f4814b425a61f802a2572, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733324282087 2024-12-04T14:58:03,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742232_1408 (size=12409) 2024-12-04T14:58:03,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742233_1409 (size=12151) 2024-12-04T14:58:03,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:03,968 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/c60fb86da5f94e0385cfcae98a07ceda 2024-12-04T14:58:03,971 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:03,971 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/e30eb49c2da34c8595c2e655449f4b38 is 50, key is test_row_0/C:col10/1733324282087/Put/seqid=0 2024-12-04T14:58:04,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/e95cf47ba9f143d2934628f28a769681 is 50, key is test_row_0/B:col10/1733324283320/Put/seqid=0 2024-12-04T14:58:04,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742234_1410 (size=12409) 2024-12-04T14:58:04,021 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/e30eb49c2da34c8595c2e655449f4b38 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e30eb49c2da34c8595c2e655449f4b38 2024-12-04T14:58:04,026 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/C of 031e210d034f5b12ccb5e29a336d9de0 into e30eb49c2da34c8595c2e655449f4b38(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:04,027 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:04,027 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/C, priority=13, startTime=1733324283832; duration=0sec 2024-12-04T14:58:04,027 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:04,027 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:C 2024-12-04T14:58:04,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324344015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324344017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742235_1411 (size=12151) 2024-12-04T14:58:04,036 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/e95cf47ba9f143d2934628f28a769681 2024-12-04T14:58:04,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324344026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/48437bec304442b08c4c6788e45c22b8 is 50, key is test_row_0/C:col10/1733324283320/Put/seqid=0 2024-12-04T14:58:04,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742236_1412 (size=12151) 2024-12-04T14:58:04,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324344135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324344135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324344144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,359 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/939d04133a7a40d79a8ec19cd3b49dcc as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/939d04133a7a40d79a8ec19cd3b49dcc 2024-12-04T14:58:04,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324344355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324344355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,364 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/A of 031e210d034f5b12ccb5e29a336d9de0 into 939d04133a7a40d79a8ec19cd3b49dcc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:04,364 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:04,364 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/A, priority=13, startTime=1733324283831; duration=0sec 2024-12-04T14:58:04,364 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:04,364 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:A 2024-12-04T14:58:04,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324344365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T14:58:04,468 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/48437bec304442b08c4c6788e45c22b8 2024-12-04T14:58:04,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/c60fb86da5f94e0385cfcae98a07ceda as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c60fb86da5f94e0385cfcae98a07ceda 2024-12-04T14:58:04,480 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c60fb86da5f94e0385cfcae98a07ceda, entries=150, sequenceid=152, filesize=11.9 K 2024-12-04T14:58:04,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/e95cf47ba9f143d2934628f28a769681 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e95cf47ba9f143d2934628f28a769681 2024-12-04T14:58:04,486 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e95cf47ba9f143d2934628f28a769681, entries=150, sequenceid=152, filesize=11.9 K 2024-12-04T14:58:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/48437bec304442b08c4c6788e45c22b8 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/48437bec304442b08c4c6788e45c22b8 2024-12-04T14:58:04,492 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/48437bec304442b08c4c6788e45c22b8, entries=150, sequenceid=152, filesize=11.9 K 2024-12-04T14:58:04,494 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 031e210d034f5b12ccb5e29a336d9de0 in 615ms, sequenceid=152, compaction requested=false 2024-12-04T14:58:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-04T14:58:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-04T14:58:04,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-04T14:58:04,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2360 sec 2024-12-04T14:58:04,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.2430 sec 2024-12-04T14:58:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:04,671 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:04,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-04T14:58:04,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/473b722441f1417fb81186fa98e4cf65 is 50, key is test_row_0/A:col10/1733324284015/Put/seqid=0 2024-12-04T14:58:04,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742237_1413 (size=14541) 2024-12-04T14:58:04,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324344736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324344743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324344748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324344855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324344866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324344866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:05,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324345075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:05,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324345085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:05,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:05,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324345085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:05,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:05,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/473b722441f1417fb81186fa98e4cf65 2024-12-04T14:58:05,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/a445c59b4d494f75bc0f76aa0cb9faf7 is 50, key is test_row_0/B:col10/1733324284015/Put/seqid=0 2024-12-04T14:58:05,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742238_1414 (size=12151) 2024-12-04T14:58:05,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/a445c59b4d494f75bc0f76aa0cb9faf7 2024-12-04T14:58:05,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/f38b04859b554e6d8c629fc1e342d2a8 is 50, key is test_row_0/C:col10/1733324284015/Put/seqid=0 2024-12-04T14:58:05,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742239_1415 (size=12151) 2024-12-04T14:58:05,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324345395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:05,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324345405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:05,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324345405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T14:58:05,415 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:05,416 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-04T14:58:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T14:58:05,419 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:05,420 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:05,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T14:58:05,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T14:58:05,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:05,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:05,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/f38b04859b554e6d8c629fc1e342d2a8 2024-12-04T14:58:05,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/473b722441f1417fb81186fa98e4cf65 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/473b722441f1417fb81186fa98e4cf65 2024-12-04T14:58:05,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/473b722441f1417fb81186fa98e4cf65, entries=200, sequenceid=174, filesize=14.2 K 2024-12-04T14:58:05,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/a445c59b4d494f75bc0f76aa0cb9faf7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/a445c59b4d494f75bc0f76aa0cb9faf7 2024-12-04T14:58:05,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/a445c59b4d494f75bc0f76aa0cb9faf7, entries=150, sequenceid=174, filesize=11.9 K 2024-12-04T14:58:05,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/f38b04859b554e6d8c629fc1e342d2a8 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/f38b04859b554e6d8c629fc1e342d2a8 2024-12-04T14:58:05,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/f38b04859b554e6d8c629fc1e342d2a8, entries=150, sequenceid=174, filesize=11.9 K 2024-12-04T14:58:05,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 031e210d034f5b12ccb5e29a336d9de0 in 949ms, sequenceid=174, compaction requested=true 2024-12-04T14:58:05,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:05,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:05,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:05,621 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:05,621 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:05,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:05,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:05,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:05,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:05,621 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:05,622 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/B is initiating minor compaction (all files) 2024-12-04T14:58:05,622 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/B in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:05,622 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e8dd08d212f740aab46fee10f7f62ee7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e95cf47ba9f143d2934628f28a769681, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/a445c59b4d494f75bc0f76aa0cb9faf7] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=35.9 K 2024-12-04T14:58:05,622 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:05,622 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/A is initiating minor compaction (all files) 2024-12-04T14:58:05,622 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/A in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:05,622 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e8dd08d212f740aab46fee10f7f62ee7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733324282087 2024-12-04T14:58:05,622 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/939d04133a7a40d79a8ec19cd3b49dcc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c60fb86da5f94e0385cfcae98a07ceda, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/473b722441f1417fb81186fa98e4cf65] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=38.2 K 2024-12-04T14:58:05,622 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e95cf47ba9f143d2934628f28a769681, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733324283286 2024-12-04T14:58:05,623 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 939d04133a7a40d79a8ec19cd3b49dcc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733324282087 2024-12-04T14:58:05,623 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting a445c59b4d494f75bc0f76aa0cb9faf7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733324284015 2024-12-04T14:58:05,623 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60fb86da5f94e0385cfcae98a07ceda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733324283286 2024-12-04T14:58:05,625 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 473b722441f1417fb81186fa98e4cf65, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733324284007 2024-12-04T14:58:05,638 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:05,638 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/d095ae610fdb49448ee94bc33e502f87 is 50, key is test_row_0/B:col10/1733324284015/Put/seqid=0 2024-12-04T14:58:05,640 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#A#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:05,641 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/0eb67333d3ff438bbf8d88d4def9aa01 is 50, key is test_row_0/A:col10/1733324284015/Put/seqid=0 2024-12-04T14:58:05,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742240_1416 (size=12561) 2024-12-04T14:58:05,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742241_1417 (size=12561) 2024-12-04T14:58:05,652 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/0eb67333d3ff438bbf8d88d4def9aa01 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0eb67333d3ff438bbf8d88d4def9aa01 2024-12-04T14:58:05,656 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/A of 031e210d034f5b12ccb5e29a336d9de0 into 0eb67333d3ff438bbf8d88d4def9aa01(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:05,656 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:05,656 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/A, priority=13, startTime=1733324285621; duration=0sec 2024-12-04T14:58:05,656 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:05,656 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:A 2024-12-04T14:58:05,656 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:05,657 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:05,657 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/C is initiating minor compaction (all files) 2024-12-04T14:58:05,657 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/C in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:05,657 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e30eb49c2da34c8595c2e655449f4b38, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/48437bec304442b08c4c6788e45c22b8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/f38b04859b554e6d8c629fc1e342d2a8] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=35.9 K 2024-12-04T14:58:05,657 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting e30eb49c2da34c8595c2e655449f4b38, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733324282087 2024-12-04T14:58:05,657 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48437bec304442b08c4c6788e45c22b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733324283286 2024-12-04T14:58:05,659 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting f38b04859b554e6d8c629fc1e342d2a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733324284015 2024-12-04T14:58:05,675 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:05,675 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/6949a2379b95427a8a80a610c04d1e1b is 50, key is test_row_0/C:col10/1733324284015/Put/seqid=0 2024-12-04T14:58:05,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742242_1418 (size=12561) 2024-12-04T14:58:05,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T14:58:05,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:05,728 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-04T14:58:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/4b2fc7bd971f494e84ae4dc2fc06e708 is 50, key is test_row_0/A:col10/1733324284732/Put/seqid=0 2024-12-04T14:58:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T14:58:05,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742243_1419 (size=12151) 2024-12-04T14:58:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:05,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324345970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324345970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:05,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324345975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T14:58:06,051 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/d095ae610fdb49448ee94bc33e502f87 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/d095ae610fdb49448ee94bc33e502f87 2024-12-04T14:58:06,056 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/B of 031e210d034f5b12ccb5e29a336d9de0 into d095ae610fdb49448ee94bc33e502f87(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:06,056 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:06,056 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/B, priority=13, startTime=1733324285621; duration=0sec 2024-12-04T14:58:06,056 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:06,056 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:B 2024-12-04T14:58:06,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324346085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:06,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324346085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324346087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:06,107 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/6949a2379b95427a8a80a610c04d1e1b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6949a2379b95427a8a80a610c04d1e1b 2024-12-04T14:58:06,110 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/C of 031e210d034f5b12ccb5e29a336d9de0 into 6949a2379b95427a8a80a610c04d1e1b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:06,110 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:06,110 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/C, priority=13, startTime=1733324285621; duration=0sec 2024-12-04T14:58:06,110 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:06,111 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:C 2024-12-04T14:58:06,144 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/4b2fc7bd971f494e84ae4dc2fc06e708 2024-12-04T14:58:06,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/5303ecb0c8f348d685f0e76348d3bf60 is 50, key is test_row_0/B:col10/1733324284732/Put/seqid=0 2024-12-04T14:58:06,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742244_1420 (size=12151) 2024-12-04T14:58:06,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324346295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:06,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324346295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:06,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324346295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:06,554 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/5303ecb0c8f348d685f0e76348d3bf60 2024-12-04T14:58:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T14:58:06,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/4fce5c4175724559a85c728ed8730cab is 50, key is test_row_0/C:col10/1733324284732/Put/seqid=0 2024-12-04T14:58:06,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742245_1421 (size=12151) 2024-12-04T14:58:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324346605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324346605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324346605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:06,967 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/4fce5c4175724559a85c728ed8730cab 2024-12-04T14:58:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/4b2fc7bd971f494e84ae4dc2fc06e708 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4b2fc7bd971f494e84ae4dc2fc06e708 2024-12-04T14:58:06,973 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4b2fc7bd971f494e84ae4dc2fc06e708, entries=150, sequenceid=192, filesize=11.9 K 2024-12-04T14:58:06,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/5303ecb0c8f348d685f0e76348d3bf60 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5303ecb0c8f348d685f0e76348d3bf60 2024-12-04T14:58:06,976 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5303ecb0c8f348d685f0e76348d3bf60, entries=150, sequenceid=192, filesize=11.9 K 2024-12-04T14:58:06,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/4fce5c4175724559a85c728ed8730cab as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4fce5c4175724559a85c728ed8730cab 2024-12-04T14:58:06,980 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4fce5c4175724559a85c728ed8730cab, entries=150, sequenceid=192, filesize=11.9 K 2024-12-04T14:58:06,981 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 031e210d034f5b12ccb5e29a336d9de0 in 1253ms, sequenceid=192, compaction requested=false 2024-12-04T14:58:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-04T14:58:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-04T14:58:06,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-04T14:58:06,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5630 sec 2024-12-04T14:58:06,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.5680 sec 2024-12-04T14:58:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:07,119 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:07,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-04T14:58:07,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/2d6ab322c8634929b2bbb00c6a4d8c94 is 50, key is test_row_0/A:col10/1733324287118/Put/seqid=0 2024-12-04T14:58:07,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742246_1422 (size=14541) 2024-12-04T14:58:07,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324347145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324347153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324347154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324347255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324347265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324347265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324347365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,368 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 031e210d034f5b12ccb5e29a336d9de0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324347384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,388 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 031e210d034f5b12ccb5e29a336d9de0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324347465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324347474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324347474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/2d6ab322c8634929b2bbb00c6a4d8c94 2024-12-04T14:58:07,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/797dd6d6a6d54f49a6b3c3151fca45fe is 50, key is test_row_0/B:col10/1733324287118/Put/seqid=0 2024-12-04T14:58:07,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742247_1423 (size=12151) 2024-12-04T14:58:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T14:58:07,565 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:07,566 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-04T14:58:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T14:58:07,567 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:07,568 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:07,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T14:58:07,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T14:58:07,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:07,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:07,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:07,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:07,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:07,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324347775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324347785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324347785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:07,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T14:58:07,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:07,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:07,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:07,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:07,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:07,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T14:58:07,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/797dd6d6a6d54f49a6b3c3151fca45fe 2024-12-04T14:58:07,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/d9092b96d1f04dac96e56e7c00e6694a is 50, key is test_row_0/C:col10/1733324287118/Put/seqid=0 2024-12-04T14:58:07,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742248_1424 (size=12151) 2024-12-04T14:58:08,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T14:58:08,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:08,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:08,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:08,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:08,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:08,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T14:58:08,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:08,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:08,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:08,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:08,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T14:58:08,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:08,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324348285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:08,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:08,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:08,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:08,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324348295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:08,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324348295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:08,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T14:58:08,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:08,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:08,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:08,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:08,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:08,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/d9092b96d1f04dac96e56e7c00e6694a 2024-12-04T14:58:08,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/2d6ab322c8634929b2bbb00c6a4d8c94 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/2d6ab322c8634929b2bbb00c6a4d8c94 2024-12-04T14:58:08,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/2d6ab322c8634929b2bbb00c6a4d8c94, entries=200, sequenceid=214, filesize=14.2 K 2024-12-04T14:58:08,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/797dd6d6a6d54f49a6b3c3151fca45fe as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/797dd6d6a6d54f49a6b3c3151fca45fe 2024-12-04T14:58:08,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/797dd6d6a6d54f49a6b3c3151fca45fe, entries=150, sequenceid=214, filesize=11.9 K 2024-12-04T14:58:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/d9092b96d1f04dac96e56e7c00e6694a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/d9092b96d1f04dac96e56e7c00e6694a 2024-12-04T14:58:08,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/d9092b96d1f04dac96e56e7c00e6694a, entries=150, sequenceid=214, filesize=11.9 K 2024-12-04T14:58:08,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 031e210d034f5b12ccb5e29a336d9de0 in 1243ms, sequenceid=214, compaction requested=true 2024-12-04T14:58:08,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:08,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:08,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:08,362 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:08,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:08,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:08,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:08,362 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:08,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:08,363 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:08,363 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:08,363 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/B is initiating minor compaction (all files) 2024-12-04T14:58:08,363 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/A is initiating minor compaction (all files) 2024-12-04T14:58:08,363 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/B in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:08,363 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/A in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:08,363 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/d095ae610fdb49448ee94bc33e502f87, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5303ecb0c8f348d685f0e76348d3bf60, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/797dd6d6a6d54f49a6b3c3151fca45fe] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.0 K 2024-12-04T14:58:08,363 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0eb67333d3ff438bbf8d88d4def9aa01, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4b2fc7bd971f494e84ae4dc2fc06e708, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/2d6ab322c8634929b2bbb00c6a4d8c94] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=38.3 K 2024-12-04T14:58:08,363 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting d095ae610fdb49448ee94bc33e502f87, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733324284015 2024-12-04T14:58:08,363 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb67333d3ff438bbf8d88d4def9aa01, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733324284015 2024-12-04T14:58:08,363 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 5303ecb0c8f348d685f0e76348d3bf60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733324284732 2024-12-04T14:58:08,364 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2fc7bd971f494e84ae4dc2fc06e708, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733324284732 2024-12-04T14:58:08,364 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 797dd6d6a6d54f49a6b3c3151fca45fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733324285967 2024-12-04T14:58:08,364 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6ab322c8634929b2bbb00c6a4d8c94, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733324285955 2024-12-04T14:58:08,369 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:08,370 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/fdd7159d8523478ea391accab5ef4f9c is 50, key is test_row_0/B:col10/1733324287118/Put/seqid=0 2024-12-04T14:58:08,370 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#A#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:08,370 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/e7f9b194c7cb4b62b5b8dfd42cf045e6 is 50, key is test_row_0/A:col10/1733324287118/Put/seqid=0 2024-12-04T14:58:08,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742249_1425 (size=12663) 2024-12-04T14:58:08,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742250_1426 (size=12663) 2024-12-04T14:58:08,391 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/e7f9b194c7cb4b62b5b8dfd42cf045e6 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/e7f9b194c7cb4b62b5b8dfd42cf045e6 2024-12-04T14:58:08,395 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/A of 031e210d034f5b12ccb5e29a336d9de0 into e7f9b194c7cb4b62b5b8dfd42cf045e6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:08,395 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:08,395 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/A, priority=13, startTime=1733324288362; duration=0sec 2024-12-04T14:58:08,395 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:08,395 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:A 2024-12-04T14:58:08,396 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:08,396 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:08,396 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/C is initiating minor compaction (all files) 2024-12-04T14:58:08,396 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/C in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:08,396 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6949a2379b95427a8a80a610c04d1e1b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4fce5c4175724559a85c728ed8730cab, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/d9092b96d1f04dac96e56e7c00e6694a] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.0 K 2024-12-04T14:58:08,397 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6949a2379b95427a8a80a610c04d1e1b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733324284015 2024-12-04T14:58:08,397 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fce5c4175724559a85c728ed8730cab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733324284732 2024-12-04T14:58:08,398 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9092b96d1f04dac96e56e7c00e6694a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733324285967 2024-12-04T14:58:08,402 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#C#compaction#362 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:08,403 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/e56cfbad5ce4486a897270860c8bd9cf is 50, key is test_row_0/C:col10/1733324287118/Put/seqid=0 2024-12-04T14:58:08,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742251_1427 (size=12663) 2024-12-04T14:58:08,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T14:58:08,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:08,484 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-04T14:58:08,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/c9fae2753cd74da28113cab30d25827d is 50, key is test_row_0/A:col10/1733324287152/Put/seqid=0 2024-12-04T14:58:08,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742252_1428 (size=12151) 2024-12-04T14:58:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T14:58:08,779 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/fdd7159d8523478ea391accab5ef4f9c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/fdd7159d8523478ea391accab5ef4f9c 2024-12-04T14:58:08,783 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/B of 031e210d034f5b12ccb5e29a336d9de0 into fdd7159d8523478ea391accab5ef4f9c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:08,783 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:08,783 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/B, priority=13, startTime=1733324288362; duration=0sec 2024-12-04T14:58:08,783 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:08,783 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:B 2024-12-04T14:58:08,813 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/e56cfbad5ce4486a897270860c8bd9cf as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e56cfbad5ce4486a897270860c8bd9cf 2024-12-04T14:58:08,816 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/C of 031e210d034f5b12ccb5e29a336d9de0 into e56cfbad5ce4486a897270860c8bd9cf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:08,816 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:08,816 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/C, priority=13, startTime=1733324288362; duration=0sec 2024-12-04T14:58:08,816 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:08,817 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:C 2024-12-04T14:58:08,892 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/c9fae2753cd74da28113cab30d25827d 2024-12-04T14:58:08,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/8a354e8249ae4801ab69c7d0fcd84e03 is 50, key is test_row_0/B:col10/1733324287152/Put/seqid=0 2024-12-04T14:58:08,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742253_1429 (size=12151) 2024-12-04T14:58:09,302 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/8a354e8249ae4801ab69c7d0fcd84e03 2024-12-04T14:58:09,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/c3b91d969fc84807aefacf416a944dc6 is 50, key is test_row_0/C:col10/1733324287152/Put/seqid=0 2024-12-04T14:58:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:09,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:09,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742254_1430 (size=12151) 2024-12-04T14:58:09,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324349342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:09,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324349343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:09,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324349350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:09,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:09,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324349455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324349455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:09,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324349464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:09,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324349665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324349665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324349675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:09,712 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/c3b91d969fc84807aefacf416a944dc6 2024-12-04T14:58:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T14:58:09,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/c9fae2753cd74da28113cab30d25827d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c9fae2753cd74da28113cab30d25827d 2024-12-04T14:58:09,721 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c9fae2753cd74da28113cab30d25827d, entries=150, sequenceid=232, filesize=11.9 K 2024-12-04T14:58:09,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/8a354e8249ae4801ab69c7d0fcd84e03 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/8a354e8249ae4801ab69c7d0fcd84e03 2024-12-04T14:58:09,725 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/8a354e8249ae4801ab69c7d0fcd84e03, entries=150, sequenceid=232, filesize=11.9 K 2024-12-04T14:58:09,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/c3b91d969fc84807aefacf416a944dc6 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c3b91d969fc84807aefacf416a944dc6 2024-12-04T14:58:09,734 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c3b91d969fc84807aefacf416a944dc6, entries=150, sequenceid=232, filesize=11.9 K 2024-12-04T14:58:09,735 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 031e210d034f5b12ccb5e29a336d9de0 in 1252ms, sequenceid=232, compaction requested=false 2024-12-04T14:58:09,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:09,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:09,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-04T14:58:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-04T14:58:09,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-04T14:58:09,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1690 sec 2024-12-04T14:58:09,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.1730 sec 2024-12-04T14:58:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:09,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:09,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-04T14:58:09,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/1cb2083e229b46b28e188c9ebced8fcc is 50, key is test_row_0/A:col10/1733324289333/Put/seqid=0 2024-12-04T14:58:10,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742255_1431 (size=14541) 2024-12-04T14:58:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324350019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324350021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324350023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324350134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324350135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324350135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324350345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324350345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324350345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/1cb2083e229b46b28e188c9ebced8fcc 2024-12-04T14:58:10,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/796d6bdf4c614c359c2ef909979e311b is 50, key is test_row_0/B:col10/1733324289333/Put/seqid=0 2024-12-04T14:58:10,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742256_1432 (size=12151) 2024-12-04T14:58:10,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324350655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324350655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324350665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:10,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:10,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/796d6bdf4c614c359c2ef909979e311b 2024-12-04T14:58:10,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/e94a1e942b994c989fe8b6f8a57fe2df is 50, key is test_row_0/C:col10/1733324289333/Put/seqid=0 2024-12-04T14:58:10,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742257_1433 (size=12151) 2024-12-04T14:58:11,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324351165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:11,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324351165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:11,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324351175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:11,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/e94a1e942b994c989fe8b6f8a57fe2df 2024-12-04T14:58:11,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/1cb2083e229b46b28e188c9ebced8fcc as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/1cb2083e229b46b28e188c9ebced8fcc 2024-12-04T14:58:11,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/1cb2083e229b46b28e188c9ebced8fcc, entries=200, sequenceid=255, filesize=14.2 K 2024-12-04T14:58:11,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/796d6bdf4c614c359c2ef909979e311b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/796d6bdf4c614c359c2ef909979e311b 2024-12-04T14:58:11,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/796d6bdf4c614c359c2ef909979e311b, entries=150, sequenceid=255, filesize=11.9 K 2024-12-04T14:58:11,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/e94a1e942b994c989fe8b6f8a57fe2df as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e94a1e942b994c989fe8b6f8a57fe2df 2024-12-04T14:58:11,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e94a1e942b994c989fe8b6f8a57fe2df, entries=150, sequenceid=255, filesize=11.9 K 2024-12-04T14:58:11,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 031e210d034f5b12ccb5e29a336d9de0 in 1267ms, sequenceid=255, compaction requested=true 2024-12-04T14:58:11,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:11,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:11,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:11,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:11,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:11,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:11,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:11,247 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:11,247 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:11,248 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:11,248 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:11,248 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/A is initiating minor compaction (all files) 2024-12-04T14:58:11,248 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/B is initiating minor compaction (all files) 2024-12-04T14:58:11,248 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/B in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:11,248 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/A in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:11,248 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/fdd7159d8523478ea391accab5ef4f9c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/8a354e8249ae4801ab69c7d0fcd84e03, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/796d6bdf4c614c359c2ef909979e311b] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.1 K 2024-12-04T14:58:11,248 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/e7f9b194c7cb4b62b5b8dfd42cf045e6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c9fae2753cd74da28113cab30d25827d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/1cb2083e229b46b28e188c9ebced8fcc] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=38.4 K 2024-12-04T14:58:11,248 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting fdd7159d8523478ea391accab5ef4f9c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733324285967 2024-12-04T14:58:11,248 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7f9b194c7cb4b62b5b8dfd42cf045e6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733324285967 2024-12-04T14:58:11,249 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a354e8249ae4801ab69c7d0fcd84e03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733324287144 2024-12-04T14:58:11,249 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 796d6bdf4c614c359c2ef909979e311b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324289333 2024-12-04T14:58:11,249 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9fae2753cd74da28113cab30d25827d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733324287144 2024-12-04T14:58:11,249 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cb2083e229b46b28e188c9ebced8fcc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324289333 2024-12-04T14:58:11,256 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:11,256 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#A#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:11,256 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/3e96b4a216184273a8387edb8511f062 is 50, key is test_row_0/A:col10/1733324289333/Put/seqid=0 2024-12-04T14:58:11,256 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/e5e20f9589d1461684993d5603b18c07 is 50, key is test_row_0/B:col10/1733324289333/Put/seqid=0 2024-12-04T14:58:11,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742258_1434 (size=12765) 2024-12-04T14:58:11,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742259_1435 (size=12765) 2024-12-04T14:58:11,664 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/e5e20f9589d1461684993d5603b18c07 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e5e20f9589d1461684993d5603b18c07 2024-12-04T14:58:11,664 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/3e96b4a216184273a8387edb8511f062 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3e96b4a216184273a8387edb8511f062 2024-12-04T14:58:11,668 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/B of 031e210d034f5b12ccb5e29a336d9de0 into e5e20f9589d1461684993d5603b18c07(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:11,668 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:11,668 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/B, priority=13, startTime=1733324291247; duration=0sec 2024-12-04T14:58:11,668 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/A of 031e210d034f5b12ccb5e29a336d9de0 into 3e96b4a216184273a8387edb8511f062(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:11,668 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:11,668 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:B 2024-12-04T14:58:11,668 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:11,668 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/A, priority=13, startTime=1733324291247; duration=0sec 2024-12-04T14:58:11,668 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:11,668 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:11,668 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:A 2024-12-04T14:58:11,669 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:11,669 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/C is initiating minor compaction (all files) 2024-12-04T14:58:11,669 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/C in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:11,669 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e56cfbad5ce4486a897270860c8bd9cf, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c3b91d969fc84807aefacf416a944dc6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e94a1e942b994c989fe8b6f8a57fe2df] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.1 K 2024-12-04T14:58:11,669 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e56cfbad5ce4486a897270860c8bd9cf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733324285967 2024-12-04T14:58:11,669 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b91d969fc84807aefacf416a944dc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733324287144 2024-12-04T14:58:11,669 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e94a1e942b994c989fe8b6f8a57fe2df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324289333 2024-12-04T14:58:11,682 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:11,682 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/8ee52e5343d34a1da26406df2db250cd is 50, key is test_row_0/C:col10/1733324289333/Put/seqid=0 2024-12-04T14:58:11,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742260_1436 (size=12765) 2024-12-04T14:58:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T14:58:11,735 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:11,735 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-04T14:58:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T14:58:11,737 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:11,739 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:11,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T14:58:11,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T14:58:11,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:11,892 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-04T14:58:11,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/283c1e0432eb466a810ebf23304318eb is 50, key is test_row_0/A:col10/1733324290003/Put/seqid=0 2024-12-04T14:58:11,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742261_1437 (size=12301) 2024-12-04T14:58:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T14:58:12,090 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/8ee52e5343d34a1da26406df2db250cd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/8ee52e5343d34a1da26406df2db250cd 2024-12-04T14:58:12,093 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/C of 031e210d034f5b12ccb5e29a336d9de0 into 8ee52e5343d34a1da26406df2db250cd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:12,093 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:12,093 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/C, priority=13, startTime=1733324291247; duration=0sec 2024-12-04T14:58:12,093 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:12,093 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:C 2024-12-04T14:58:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:12,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324352227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324352229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324352233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,300 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/283c1e0432eb466a810ebf23304318eb 2024-12-04T14:58:12,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/515c330759784097aaea9caa0fc9556d is 50, key is test_row_0/B:col10/1733324290003/Put/seqid=0 2024-12-04T14:58:12,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742262_1438 (size=12301) 2024-12-04T14:58:12,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324352345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324352345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324352346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T14:58:12,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324352555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324352555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324352555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,711 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/515c330759784097aaea9caa0fc9556d 2024-12-04T14:58:12,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/c92c08481b57437ca8a44690dab94a3a is 50, key is test_row_0/C:col10/1733324290003/Put/seqid=0 2024-12-04T14:58:12,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742263_1439 (size=12301) 2024-12-04T14:58:12,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324352865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324352866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T14:58:12,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324352875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:12,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:13,125 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/c92c08481b57437ca8a44690dab94a3a 2024-12-04T14:58:13,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/283c1e0432eb466a810ebf23304318eb as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/283c1e0432eb466a810ebf23304318eb 2024-12-04T14:58:13,131 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/283c1e0432eb466a810ebf23304318eb, entries=150, sequenceid=272, filesize=12.0 K 2024-12-04T14:58:13,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/515c330759784097aaea9caa0fc9556d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/515c330759784097aaea9caa0fc9556d 2024-12-04T14:58:13,134 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/515c330759784097aaea9caa0fc9556d, entries=150, sequenceid=272, filesize=12.0 K 2024-12-04T14:58:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/c92c08481b57437ca8a44690dab94a3a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c92c08481b57437ca8a44690dab94a3a 2024-12-04T14:58:13,138 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c92c08481b57437ca8a44690dab94a3a, entries=150, sequenceid=272, filesize=12.0 K 2024-12-04T14:58:13,139 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 031e210d034f5b12ccb5e29a336d9de0 in 1247ms, sequenceid=272, compaction requested=false 2024-12-04T14:58:13,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:13,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:13,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-04T14:58:13,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-04T14:58:13,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-04T14:58:13,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-12-04T14:58:13,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4070 sec 2024-12-04T14:58:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:13,378 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:13,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-04T14:58:13,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/81c244c86f3f4a588222afd09a9a58ae is 50, key is test_row_0/A:col10/1733324293377/Put/seqid=0 2024-12-04T14:58:13,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742264_1440 (size=17181) 2024-12-04T14:58:13,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324353405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:13,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324353405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324353405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:13,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324353515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:13,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324353515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:13,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324353524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:13,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324353725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324353725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:13,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324353735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:13,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/81c244c86f3f4a588222afd09a9a58ae 2024-12-04T14:58:13,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/f8b4bce840d9484db0e8a249fc39baee is 50, key is test_row_0/B:col10/1733324293377/Put/seqid=0 2024-12-04T14:58:13,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742265_1441 (size=12301) 2024-12-04T14:58:13,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/f8b4bce840d9484db0e8a249fc39baee 2024-12-04T14:58:13,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/1b46417b342a4d8cb3c1235199aa237c is 50, key is test_row_0/C:col10/1733324293377/Put/seqid=0 2024-12-04T14:58:13,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742266_1442 (size=12301) 2024-12-04T14:58:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T14:58:13,885 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:13,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:13,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-04T14:58:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T14:58:13,887 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:13,887 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:13,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T14:58:14,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324354035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324354035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T14:58:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:14,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:14,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:14,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324354045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T14:58:14,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:14,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:14,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:14,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:14,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:14,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/1b46417b342a4d8cb3c1235199aa237c 2024-12-04T14:58:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T14:58:14,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/81c244c86f3f4a588222afd09a9a58ae as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/81c244c86f3f4a588222afd09a9a58ae 2024-12-04T14:58:14,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/81c244c86f3f4a588222afd09a9a58ae, entries=250, sequenceid=296, filesize=16.8 K 2024-12-04T14:58:14,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/f8b4bce840d9484db0e8a249fc39baee as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f8b4bce840d9484db0e8a249fc39baee 2024-12-04T14:58:14,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f8b4bce840d9484db0e8a249fc39baee, entries=150, sequenceid=296, filesize=12.0 K 2024-12-04T14:58:14,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/1b46417b342a4d8cb3c1235199aa237c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1b46417b342a4d8cb3c1235199aa237c 2024-12-04T14:58:14,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1b46417b342a4d8cb3c1235199aa237c, entries=150, sequenceid=296, filesize=12.0 K 2024-12-04T14:58:14,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 031e210d034f5b12ccb5e29a336d9de0 in 840ms, sequenceid=296, compaction requested=true 2024-12-04T14:58:14,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:14,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:14,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:14,219 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:14,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:14,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:14,219 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:14,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:14,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:14,220 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42247 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:14,220 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:14,220 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/B is initiating minor compaction (all files) 2024-12-04T14:58:14,220 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/A is initiating minor compaction (all files) 2024-12-04T14:58:14,220 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/A in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:14,220 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/B in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:14,220 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3e96b4a216184273a8387edb8511f062, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/283c1e0432eb466a810ebf23304318eb, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/81c244c86f3f4a588222afd09a9a58ae] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=41.3 K 2024-12-04T14:58:14,220 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e5e20f9589d1461684993d5603b18c07, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/515c330759784097aaea9caa0fc9556d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f8b4bce840d9484db0e8a249fc39baee] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.5 K 2024-12-04T14:58:14,220 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e5e20f9589d1461684993d5603b18c07, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324289333 2024-12-04T14:58:14,220 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e96b4a216184273a8387edb8511f062, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324289333 2024-12-04T14:58:14,220 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 283c1e0432eb466a810ebf23304318eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733324290003 2024-12-04T14:58:14,220 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 515c330759784097aaea9caa0fc9556d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733324290003 2024-12-04T14:58:14,220 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81c244c86f3f4a588222afd09a9a58ae, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324292217 2024-12-04T14:58:14,221 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting f8b4bce840d9484db0e8a249fc39baee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324292225 2024-12-04T14:58:14,225 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#B#compaction#378 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:14,226 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/3e38784b91ad49318f5bf0b7a62b5840 is 50, key is test_row_0/B:col10/1733324293377/Put/seqid=0 2024-12-04T14:58:14,226 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:14,227 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/ca609af3838a4850838d4c9f34174d53 is 50, key is test_row_0/A:col10/1733324293377/Put/seqid=0 2024-12-04T14:58:14,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742267_1443 (size=13017) 2024-12-04T14:58:14,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742268_1444 (size=13017) 2024-12-04T14:58:14,239 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/3e38784b91ad49318f5bf0b7a62b5840 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/3e38784b91ad49318f5bf0b7a62b5840 2024-12-04T14:58:14,242 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/B of 031e210d034f5b12ccb5e29a336d9de0 into 3e38784b91ad49318f5bf0b7a62b5840(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:14,242 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:14,242 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/B, priority=13, startTime=1733324294219; duration=0sec 2024-12-04T14:58:14,242 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:14,242 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:B 2024-12-04T14:58:14,243 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:14,243 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:14,243 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/C is initiating minor compaction (all files) 2024-12-04T14:58:14,243 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/C in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:14,243 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/8ee52e5343d34a1da26406df2db250cd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c92c08481b57437ca8a44690dab94a3a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1b46417b342a4d8cb3c1235199aa237c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.5 K 2024-12-04T14:58:14,243 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ee52e5343d34a1da26406df2db250cd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733324289333 2024-12-04T14:58:14,244 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c92c08481b57437ca8a44690dab94a3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733324290003 2024-12-04T14:58:14,244 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b46417b342a4d8cb3c1235199aa237c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324292225 2024-12-04T14:58:14,248 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#C#compaction#380 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:14,249 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/e5b196b81b19437aa6a7e3d1115187b1 is 50, key is test_row_0/C:col10/1733324293377/Put/seqid=0 2024-12-04T14:58:14,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742269_1445 (size=13017) 2024-12-04T14:58:14,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T14:58:14,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:14,345 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-04T14:58:14,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/ce149c976dfd49779b4c3707bb5ad64e is 50, key is test_row_0/A:col10/1733324293394/Put/seqid=0 2024-12-04T14:58:14,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742270_1446 (size=12301) 2024-12-04T14:58:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T14:58:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:14,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:14,567 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x146f54dbd02c06d1 with lease ID 0xa8d4088707cbc766: from storage DS-4a7129ce-3f0f-4ff0-b959-d3d1a01d4347 node DatanodeRegistration(127.0.0.1:35153, datanodeUuid=d3ec1071-8fa2-49ad-85bb-9771668d1f80, infoPort=46813, infoSecurePort=0, ipcPort=33625, storageInfo=lv=-57;cid=testClusterID;nsid=1374277302;c=1733324154112), blocks: 208, hasStaleStorage: false, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-04T14:58:14,569 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x146f54dbd02c06d1 with lease ID 0xa8d4088707cbc766: from storage DS-040fde0d-b820-4527-8bbe-6c6f49028344 node DatanodeRegistration(127.0.0.1:35153, datanodeUuid=d3ec1071-8fa2-49ad-85bb-9771668d1f80, infoPort=46813, infoSecurePort=0, ipcPort=33625, storageInfo=lv=-57;cid=testClusterID;nsid=1374277302;c=1733324154112), blocks: 218, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T14:58:14,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324354608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324354608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324354609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,640 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/ca609af3838a4850838d4c9f34174d53 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ca609af3838a4850838d4c9f34174d53 2024-12-04T14:58:14,644 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/A of 031e210d034f5b12ccb5e29a336d9de0 into ca609af3838a4850838d4c9f34174d53(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:14,644 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:14,644 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/A, priority=13, startTime=1733324294219; duration=0sec 2024-12-04T14:58:14,644 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:14,644 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:A 2024-12-04T14:58:14,656 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/e5b196b81b19437aa6a7e3d1115187b1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e5b196b81b19437aa6a7e3d1115187b1 2024-12-04T14:58:14,660 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/C of 031e210d034f5b12ccb5e29a336d9de0 into e5b196b81b19437aa6a7e3d1115187b1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:14,660 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:14,660 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/C, priority=13, startTime=1733324294219; duration=0sec 2024-12-04T14:58:14,660 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:14,660 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:C 2024-12-04T14:58:14,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324354724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324354725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324354725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,752 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/ce149c976dfd49779b4c3707bb5ad64e 2024-12-04T14:58:14,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/1445eeaf06f0494bbc3d8979debee7c5 is 50, key is test_row_0/B:col10/1733324293394/Put/seqid=0 2024-12-04T14:58:14,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742271_1447 (size=12301) 2024-12-04T14:58:14,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324354935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324354935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324354935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T14:58:15,164 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/1445eeaf06f0494bbc3d8979debee7c5 2024-12-04T14:58:15,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/ef130139226746ffaac07cee5d021d28 is 50, key is test_row_0/C:col10/1733324293394/Put/seqid=0 2024-12-04T14:58:15,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742272_1448 (size=12301) 2024-12-04T14:58:15,186 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/ef130139226746ffaac07cee5d021d28 2024-12-04T14:58:15,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/ce149c976dfd49779b4c3707bb5ad64e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ce149c976dfd49779b4c3707bb5ad64e 2024-12-04T14:58:15,196 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ce149c976dfd49779b4c3707bb5ad64e, entries=150, sequenceid=310, filesize=12.0 K 2024-12-04T14:58:15,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/1445eeaf06f0494bbc3d8979debee7c5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/1445eeaf06f0494bbc3d8979debee7c5 2024-12-04T14:58:15,207 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/1445eeaf06f0494bbc3d8979debee7c5, entries=150, sequenceid=310, filesize=12.0 K 2024-12-04T14:58:15,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/ef130139226746ffaac07cee5d021d28 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/ef130139226746ffaac07cee5d021d28 2024-12-04T14:58:15,213 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/ef130139226746ffaac07cee5d021d28, entries=150, sequenceid=310, filesize=12.0 K 2024-12-04T14:58:15,213 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 031e210d034f5b12ccb5e29a336d9de0 in 869ms, sequenceid=310, compaction requested=false 2024-12-04T14:58:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-04T14:58:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-04T14:58:15,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-04T14:58:15,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3280 sec 2024-12-04T14:58:15,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.3350 sec 2024-12-04T14:58:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:15,248 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:15,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:58:15,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/16b1eeab787d4541bc6928fb32bd3f2f is 50, key is test_row_0/A:col10/1733324294589/Put/seqid=0 2024-12-04T14:58:15,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742273_1449 (size=14741) 2024-12-04T14:58:15,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324355266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324355266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324355267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324355375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324355385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324355385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324355596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324355596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324355596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/16b1eeab787d4541bc6928fb32bd3f2f 2024-12-04T14:58:15,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/6318b1766eec4b8fb7ebcf017bc095a1 is 50, key is test_row_0/B:col10/1733324294589/Put/seqid=0 2024-12-04T14:58:15,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742274_1450 (size=12301) 2024-12-04T14:58:15,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/6318b1766eec4b8fb7ebcf017bc095a1 2024-12-04T14:58:15,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/6d0e710ae9bc407f9ea8c8df6eccc25b is 50, key is test_row_0/C:col10/1733324294589/Put/seqid=0 2024-12-04T14:58:15,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742275_1451 (size=12301) 2024-12-04T14:58:15,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324355905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324355905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:15,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324355914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T14:58:16,035 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:16,035 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-04T14:58:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T14:58:16,037 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:16,037 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:16,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:16,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/6d0e710ae9bc407f9ea8c8df6eccc25b 2024-12-04T14:58:16,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/16b1eeab787d4541bc6928fb32bd3f2f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/16b1eeab787d4541bc6928fb32bd3f2f 2024-12-04T14:58:16,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/16b1eeab787d4541bc6928fb32bd3f2f, entries=200, sequenceid=337, filesize=14.4 K 2024-12-04T14:58:16,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/6318b1766eec4b8fb7ebcf017bc095a1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/6318b1766eec4b8fb7ebcf017bc095a1 2024-12-04T14:58:16,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/6318b1766eec4b8fb7ebcf017bc095a1, entries=150, sequenceid=337, filesize=12.0 K 2024-12-04T14:58:16,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/6d0e710ae9bc407f9ea8c8df6eccc25b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6d0e710ae9bc407f9ea8c8df6eccc25b 2024-12-04T14:58:16,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6d0e710ae9bc407f9ea8c8df6eccc25b, entries=150, sequenceid=337, filesize=12.0 K 2024-12-04T14:58:16,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 031e210d034f5b12ccb5e29a336d9de0 in 853ms, sequenceid=337, compaction requested=true 2024-12-04T14:58:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:16,101 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T14:58:16,102 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:16,102 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/A is initiating minor compaction (all files) 2024-12-04T14:58:16,102 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/A in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:16,102 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ca609af3838a4850838d4c9f34174d53, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ce149c976dfd49779b4c3707bb5ad64e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/16b1eeab787d4541bc6928fb32bd3f2f] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=39.1 K 2024-12-04T14:58:16,102 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca609af3838a4850838d4c9f34174d53, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324292225 2024-12-04T14:58:16,103 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:16,103 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce149c976dfd49779b4c3707bb5ad64e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733324293394 2024-12-04T14:58:16,103 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16b1eeab787d4541bc6928fb32bd3f2f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733324294579 2024-12-04T14:58:16,103 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:16,104 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/B is initiating minor compaction (all files) 2024-12-04T14:58:16,104 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/B in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:16,104 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/3e38784b91ad49318f5bf0b7a62b5840, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/1445eeaf06f0494bbc3d8979debee7c5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/6318b1766eec4b8fb7ebcf017bc095a1] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.7 K 2024-12-04T14:58:16,104 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e38784b91ad49318f5bf0b7a62b5840, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324292225 2024-12-04T14:58:16,104 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 1445eeaf06f0494bbc3d8979debee7c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733324293394 2024-12-04T14:58:16,105 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 6318b1766eec4b8fb7ebcf017bc095a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733324294579 2024-12-04T14:58:16,109 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:16,110 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/02bea73caca34f249241bd306b89024a is 50, key is test_row_0/A:col10/1733324294589/Put/seqid=0 2024-12-04T14:58:16,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742276_1452 (size=13119) 2024-12-04T14:58:16,123 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:16,123 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/bd38638ff82342aeaa305a9a1b9898c4 is 50, key is test_row_0/B:col10/1733324294589/Put/seqid=0 2024-12-04T14:58:16,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742277_1453 (size=13119) 2024-12-04T14:58:16,128 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/02bea73caca34f249241bd306b89024a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/02bea73caca34f249241bd306b89024a 2024-12-04T14:58:16,130 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/bd38638ff82342aeaa305a9a1b9898c4 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/bd38638ff82342aeaa305a9a1b9898c4 2024-12-04T14:58:16,132 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/A of 031e210d034f5b12ccb5e29a336d9de0 into 02bea73caca34f249241bd306b89024a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:16,132 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:16,132 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/A, priority=13, startTime=1733324296101; duration=0sec 2024-12-04T14:58:16,132 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:16,132 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:A 2024-12-04T14:58:16,132 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:16,133 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:16,133 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/C is initiating minor compaction (all files) 2024-12-04T14:58:16,133 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/C in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:16,133 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e5b196b81b19437aa6a7e3d1115187b1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/ef130139226746ffaac07cee5d021d28, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6d0e710ae9bc407f9ea8c8df6eccc25b] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.7 K 2024-12-04T14:58:16,134 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5b196b81b19437aa6a7e3d1115187b1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733324292225 2024-12-04T14:58:16,134 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef130139226746ffaac07cee5d021d28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733324293394 2024-12-04T14:58:16,135 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d0e710ae9bc407f9ea8c8df6eccc25b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733324294579 2024-12-04T14:58:16,137 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/B of 031e210d034f5b12ccb5e29a336d9de0 into bd38638ff82342aeaa305a9a1b9898c4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:16,137 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:16,137 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/B, priority=13, startTime=1733324296101; duration=0sec 2024-12-04T14:58:16,137 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:16,137 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:B 2024-12-04T14:58:16,141 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:16,142 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/863289ba8a6542e4861edf251e113eeb is 50, key is test_row_0/C:col10/1733324294589/Put/seqid=0 2024-12-04T14:58:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T14:58:16,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742278_1454 (size=13119) 2024-12-04T14:58:16,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T14:58:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:16,190 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:58:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/4eea8e286fec429182e52f34a7eb69e7 is 50, key is test_row_0/A:col10/1733324295266/Put/seqid=0 2024-12-04T14:58:16,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742279_1455 (size=12301) 2024-12-04T14:58:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T14:58:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:16,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. as already flushing 2024-12-04T14:58:16,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324356460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:16,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324356464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:16,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324356466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:16,559 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/863289ba8a6542e4861edf251e113eeb as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/863289ba8a6542e4861edf251e113eeb 2024-12-04T14:58:16,562 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/C of 031e210d034f5b12ccb5e29a336d9de0 into 863289ba8a6542e4861edf251e113eeb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:16,562 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:16,563 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/C, priority=13, startTime=1733324296101; duration=0sec 2024-12-04T14:58:16,563 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:16,563 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:C 2024-12-04T14:58:16,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324356574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:16,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324356574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:16,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324356575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:16,596 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/4eea8e286fec429182e52f34a7eb69e7 2024-12-04T14:58:16,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/b4679369c4694aea967be3f9db710422 is 50, key is test_row_0/B:col10/1733324295266/Put/seqid=0 2024-12-04T14:58:16,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742280_1456 (size=12301) 2024-12-04T14:58:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T14:58:16,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324356785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:16,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324356785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324356785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:16,821 INFO [Thread-1750 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1750. 2024-12-04T14:58:16,821 DEBUG [Thread-1750 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:16,821 DEBUG [Thread-1750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:16,821 DEBUG [Thread-1750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:16,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:16,825 INFO [Thread-1746 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1746. 2024-12-04T14:58:16,825 DEBUG [Thread-1746 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:16,825 DEBUG [Thread-1746 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:16,825 DEBUG [Thread-1746 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:16,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:16,826 INFO [Thread-1748 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1748. 2024-12-04T14:58:16,826 DEBUG [Thread-1748 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:16,826 DEBUG [Thread-1748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:16,826 DEBUG [Thread-1748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:16,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:16,827 INFO [Thread-1744 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1744. 2024-12-04T14:58:16,827 DEBUG [Thread-1744 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:16,827 DEBUG [Thread-1744 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:16,827 DEBUG [Thread-1744 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:16,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:16,830 INFO [Thread-1742 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1742. 2024-12-04T14:58:16,830 DEBUG [Thread-1742 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:16,830 DEBUG [Thread-1742 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:16,830 DEBUG [Thread-1742 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:16,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:17,005 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/b4679369c4694aea967be3f9db710422 2024-12-04T14:58:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/4e0e72766cd4497399208af8357a5d6b is 50, key is test_row_0/C:col10/1733324295266/Put/seqid=0 2024-12-04T14:58:17,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742281_1457 (size=12301) 2024-12-04T14:58:17,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:17,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:17,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59538 deadline: 1733324357095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59494 deadline: 1733324357096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59540 deadline: 1733324357095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T14:58:17,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:17,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59504 deadline: 1733324357415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59522 deadline: 1733324357416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:17,417 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 031e210d034f5b12ccb5e29a336d9de0 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:17,417 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in 031e210d034f5b12ccb5e29a336d9de0 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 , the old value is region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., hostname=d214edae7bd2,34477,1733324158096, seqNum=2 because the exception is null or not the one we care about 2024-12-04T14:58:17,422 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/4e0e72766cd4497399208af8357a5d6b 2024-12-04T14:58:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/4eea8e286fec429182e52f34a7eb69e7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4eea8e286fec429182e52f34a7eb69e7 2024-12-04T14:58:17,436 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4eea8e286fec429182e52f34a7eb69e7, entries=150, sequenceid=350, filesize=12.0 K 2024-12-04T14:58:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/b4679369c4694aea967be3f9db710422 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b4679369c4694aea967be3f9db710422 2024-12-04T14:58:17,445 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b4679369c4694aea967be3f9db710422, entries=150, sequenceid=350, filesize=12.0 K 2024-12-04T14:58:17,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/4e0e72766cd4497399208af8357a5d6b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4e0e72766cd4497399208af8357a5d6b 2024-12-04T14:58:17,452 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4e0e72766cd4497399208af8357a5d6b, entries=150, sequenceid=350, filesize=12.0 K 2024-12-04T14:58:17,452 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 031e210d034f5b12ccb5e29a336d9de0 in 1262ms, sequenceid=350, compaction requested=false 2024-12-04T14:58:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:17,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:17,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-04T14:58:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-04T14:58:17,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-04T14:58:17,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-12-04T14:58:17,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4220 sec 2024-12-04T14:58:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:17,606 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:17,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-12-04T14:58:17,606 INFO [Thread-1739 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1739. 2024-12-04T14:58:17,606 INFO [Thread-1731 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1731. 2024-12-04T14:58:17,606 INFO [Thread-1737 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1737. 2024-12-04T14:58:17,606 DEBUG [Thread-1737 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:17,606 DEBUG [Thread-1739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:17,606 DEBUG [Thread-1731 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:17,607 DEBUG [Thread-1737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:17,607 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:17,607 DEBUG [Thread-1731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:17,607 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:17,607 DEBUG [Thread-1737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:17,607 DEBUG [Thread-1731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:17,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:17,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:17,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:17,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/0ac505ae0df540c98dd258aef1ec4db1 is 50, key is test_row_0/A:col10/1733324297605/Put/seqid=0 2024-12-04T14:58:17,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742282_1458 (size=12301) 2024-12-04T14:58:18,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/0ac505ae0df540c98dd258aef1ec4db1 2024-12-04T14:58:18,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/b2367ee528aa4754a16ba837164ae0a5 is 50, key is test_row_0/B:col10/1733324297605/Put/seqid=0 2024-12-04T14:58:18,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742283_1459 (size=12301) 2024-12-04T14:58:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T14:58:18,185 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:18,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/b2367ee528aa4754a16ba837164ae0a5 2024-12-04T14:58:18,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/72035906835f4c7bb69b8aca7becef69 is 50, key is test_row_0/C:col10/1733324297605/Put/seqid=0 2024-12-04T14:58:18,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742284_1460 (size=12301) 2024-12-04T14:58:18,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/72035906835f4c7bb69b8aca7becef69 2024-12-04T14:58:18,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/0ac505ae0df540c98dd258aef1ec4db1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0ac505ae0df540c98dd258aef1ec4db1 2024-12-04T14:58:18,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0ac505ae0df540c98dd258aef1ec4db1, entries=150, sequenceid=379, filesize=12.0 K 2024-12-04T14:58:18,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/b2367ee528aa4754a16ba837164ae0a5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b2367ee528aa4754a16ba837164ae0a5 2024-12-04T14:58:18,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b2367ee528aa4754a16ba837164ae0a5, entries=150, sequenceid=379, filesize=12.0 K 2024-12-04T14:58:18,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/72035906835f4c7bb69b8aca7becef69 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/72035906835f4c7bb69b8aca7becef69 2024-12-04T14:58:18,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/72035906835f4c7bb69b8aca7becef69, entries=150, sequenceid=379, filesize=12.0 K 2024-12-04T14:58:18,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=0 B/0 for 031e210d034f5b12ccb5e29a336d9de0 in 1266ms, sequenceid=379, compaction requested=true 2024-12-04T14:58:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:18,872 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 031e210d034f5b12ccb5e29a336d9de0:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:18,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:18,872 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:18,873 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:18,873 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:18,873 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/B is initiating minor compaction (all files) 2024-12-04T14:58:18,873 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/A is initiating minor compaction (all files) 2024-12-04T14:58:18,873 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/B in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:18,873 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/A in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:18,873 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/bd38638ff82342aeaa305a9a1b9898c4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b4679369c4694aea967be3f9db710422, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b2367ee528aa4754a16ba837164ae0a5] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.8 K 2024-12-04T14:58:18,873 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/02bea73caca34f249241bd306b89024a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4eea8e286fec429182e52f34a7eb69e7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0ac505ae0df540c98dd258aef1ec4db1] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.8 K 2024-12-04T14:58:18,873 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting bd38638ff82342aeaa305a9a1b9898c4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733324294579 2024-12-04T14:58:18,873 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02bea73caca34f249241bd306b89024a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733324294579 2024-12-04T14:58:18,874 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b4679369c4694aea967be3f9db710422, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733324295259 2024-12-04T14:58:18,874 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eea8e286fec429182e52f34a7eb69e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733324295259 2024-12-04T14:58:18,874 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b2367ee528aa4754a16ba837164ae0a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733324297605 2024-12-04T14:58:18,874 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ac505ae0df540c98dd258aef1ec4db1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733324297605 2024-12-04T14:58:18,879 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:18,879 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:18,879 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/f95a7615de3c493c829e44ee7667447a is 50, key is test_row_0/B:col10/1733324297605/Put/seqid=0 2024-12-04T14:58:18,879 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/645f2d6923c34b95a87726c87a76bc34 is 50, key is test_row_0/A:col10/1733324297605/Put/seqid=0 2024-12-04T14:58:18,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742285_1461 (size=13221) 2024-12-04T14:58:18,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742286_1462 (size=13221) 2024-12-04T14:58:19,286 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/645f2d6923c34b95a87726c87a76bc34 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/645f2d6923c34b95a87726c87a76bc34 2024-12-04T14:58:19,286 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/f95a7615de3c493c829e44ee7667447a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f95a7615de3c493c829e44ee7667447a 2024-12-04T14:58:19,290 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/B of 031e210d034f5b12ccb5e29a336d9de0 into f95a7615de3c493c829e44ee7667447a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:19,290 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/A of 031e210d034f5b12ccb5e29a336d9de0 into 645f2d6923c34b95a87726c87a76bc34(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:19,290 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:19,290 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:19,290 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/B, priority=13, startTime=1733324298872; duration=0sec 2024-12-04T14:58:19,290 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/A, priority=13, startTime=1733324298872; duration=0sec 2024-12-04T14:58:19,290 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:19,290 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:B 2024-12-04T14:58:19,290 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:19,290 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:A 2024-12-04T14:58:19,290 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:19,291 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:19,291 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): 031e210d034f5b12ccb5e29a336d9de0/C is initiating minor compaction (all files) 2024-12-04T14:58:19,291 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 031e210d034f5b12ccb5e29a336d9de0/C in TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:19,291 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/863289ba8a6542e4861edf251e113eeb, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4e0e72766cd4497399208af8357a5d6b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/72035906835f4c7bb69b8aca7becef69] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp, totalSize=36.8 K 2024-12-04T14:58:19,291 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 863289ba8a6542e4861edf251e113eeb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733324294579 2024-12-04T14:58:19,291 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e0e72766cd4497399208af8357a5d6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733324295259 2024-12-04T14:58:19,292 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 72035906835f4c7bb69b8aca7becef69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733324297605 2024-12-04T14:58:19,297 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 031e210d034f5b12ccb5e29a336d9de0#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:19,297 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/3ff61f588ad644d9b2f7d3dec3e3cc26 is 50, key is test_row_0/C:col10/1733324297605/Put/seqid=0 2024-12-04T14:58:19,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742287_1463 (size=13221) 2024-12-04T14:58:19,705 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/3ff61f588ad644d9b2f7d3dec3e3cc26 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/3ff61f588ad644d9b2f7d3dec3e3cc26 2024-12-04T14:58:19,709 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 031e210d034f5b12ccb5e29a336d9de0/C of 031e210d034f5b12ccb5e29a336d9de0 into 3ff61f588ad644d9b2f7d3dec3e3cc26(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:19,709 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 031e210d034f5b12ccb5e29a336d9de0: 2024-12-04T14:58:19,709 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., storeName=031e210d034f5b12ccb5e29a336d9de0/C, priority=13, startTime=1733324298872; duration=0sec 2024-12-04T14:58:19,709 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:19,709 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 031e210d034f5b12ccb5e29a336d9de0:C 2024-12-04T14:58:26,578 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:58:27,437 INFO [Thread-1735 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1735. 2024-12-04T14:58:27,437 DEBUG [Thread-1735 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:27,437 DEBUG [Thread-1735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:27,438 DEBUG [Thread-1735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:27,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:27,446 INFO [Thread-1733 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1733. 2024-12-04T14:58:27,446 DEBUG [Thread-1733 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:27,446 DEBUG [Thread-1733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:27,446 DEBUG [Thread-1733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:27,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1848 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5544 rows 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1854 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5562 rows 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1855 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5562 rows 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1839 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5517 rows 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1842 2024-12-04T14:58:27,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5523 rows 2024-12-04T14:58:27,447 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:58:27,447 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:58:27,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:27,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:27,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:27,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T14:58:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T14:58:27,450 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324307450"}]},"ts":"1733324307450"} 2024-12-04T14:58:27,451 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T14:58:27,451 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T14:58:27,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:58:27,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=031e210d034f5b12ccb5e29a336d9de0, UNASSIGN}] 2024-12-04T14:58:27,452 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=031e210d034f5b12ccb5e29a336d9de0, UNASSIGN 2024-12-04T14:58:27,453 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=031e210d034f5b12ccb5e29a336d9de0, regionState=CLOSING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:27,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=031e210d034f5b12ccb5e29a336d9de0, UNASSIGN because future has completed 2024-12-04T14:58:27,454 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:58:27,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:58:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T14:58:27,607 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:27,607 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:58:27,607 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 031e210d034f5b12ccb5e29a336d9de0, disabling compactions & flushes 2024-12-04T14:58:27,607 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:27,607 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:27,607 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. after waiting 0 ms 2024-12-04T14:58:27,607 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:27,607 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 031e210d034f5b12ccb5e29a336d9de0 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-12-04T14:58:27,615 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/abf238246ba74bfaa733e9a312318687 is 50, key is test_row_1/A:col10/1733324307445/Put/seqid=0 2024-12-04T14:58:27,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742288_1464 (size=7415) 2024-12-04T14:58:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T14:58:28,022 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/abf238246ba74bfaa733e9a312318687 2024-12-04T14:58:28,034 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/8d167d7915d8481da3b0ea8140811622 is 50, key is test_row_1/B:col10/1733324307445/Put/seqid=0 2024-12-04T14:58:28,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742289_1465 (size=7415) 2024-12-04T14:58:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T14:58:28,439 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/8d167d7915d8481da3b0ea8140811622 2024-12-04T14:58:28,445 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/fbd58683cda34bdc95ad159bb31d8dd1 is 50, key is test_row_1/C:col10/1733324307445/Put/seqid=0 2024-12-04T14:58:28,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742290_1466 (size=7415) 2024-12-04T14:58:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T14:58:28,849 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/fbd58683cda34bdc95ad159bb31d8dd1 2024-12-04T14:58:28,852 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/A/abf238246ba74bfaa733e9a312318687 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/abf238246ba74bfaa733e9a312318687 2024-12-04T14:58:28,856 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/abf238246ba74bfaa733e9a312318687, entries=50, sequenceid=387, filesize=7.2 K 2024-12-04T14:58:28,856 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/B/8d167d7915d8481da3b0ea8140811622 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/8d167d7915d8481da3b0ea8140811622 2024-12-04T14:58:28,860 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/8d167d7915d8481da3b0ea8140811622, entries=50, sequenceid=387, filesize=7.2 K 2024-12-04T14:58:28,860 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/.tmp/C/fbd58683cda34bdc95ad159bb31d8dd1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/fbd58683cda34bdc95ad159bb31d8dd1 2024-12-04T14:58:28,863 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/fbd58683cda34bdc95ad159bb31d8dd1, entries=50, sequenceid=387, filesize=7.2 K 2024-12-04T14:58:28,864 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 031e210d034f5b12ccb5e29a336d9de0 in 1257ms, sequenceid=387, compaction requested=false 2024-12-04T14:58:28,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/63c8e2686dec41c2983ef6c3f7b934b1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/fb5f9733fc1e47e7b33aa1bd9d8176a1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/73740ed1833f4cc09a7c3ae5644e776f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/185f7bb1fdb94f3e911487efef0ca85d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3eb6ad233cfa40faa2b797cfe5c25db6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/36df617fa6cf4fa3a2a19894435b0df6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/094a60d3e78c48fbb4a07456d16aac0e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/bb88141640124429867e4114ae48a83b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/939d04133a7a40d79a8ec19cd3b49dcc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4614de8e33e14b14baba09f5bb516df2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c60fb86da5f94e0385cfcae98a07ceda, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/473b722441f1417fb81186fa98e4cf65, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0eb67333d3ff438bbf8d88d4def9aa01, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4b2fc7bd971f494e84ae4dc2fc06e708, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/2d6ab322c8634929b2bbb00c6a4d8c94, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/e7f9b194c7cb4b62b5b8dfd42cf045e6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c9fae2753cd74da28113cab30d25827d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/1cb2083e229b46b28e188c9ebced8fcc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3e96b4a216184273a8387edb8511f062, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/283c1e0432eb466a810ebf23304318eb, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/81c244c86f3f4a588222afd09a9a58ae, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ca609af3838a4850838d4c9f34174d53, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ce149c976dfd49779b4c3707bb5ad64e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/16b1eeab787d4541bc6928fb32bd3f2f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/02bea73caca34f249241bd306b89024a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4eea8e286fec429182e52f34a7eb69e7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0ac505ae0df540c98dd258aef1ec4db1] to archive 2024-12-04T14:58:28,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:58:28,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/63c8e2686dec41c2983ef6c3f7b934b1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/63c8e2686dec41c2983ef6c3f7b934b1 2024-12-04T14:58:28,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/fb5f9733fc1e47e7b33aa1bd9d8176a1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/fb5f9733fc1e47e7b33aa1bd9d8176a1 2024-12-04T14:58:28,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/73740ed1833f4cc09a7c3ae5644e776f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/73740ed1833f4cc09a7c3ae5644e776f 2024-12-04T14:58:28,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/185f7bb1fdb94f3e911487efef0ca85d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/185f7bb1fdb94f3e911487efef0ca85d 2024-12-04T14:58:28,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3eb6ad233cfa40faa2b797cfe5c25db6 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3eb6ad233cfa40faa2b797cfe5c25db6 2024-12-04T14:58:28,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/36df617fa6cf4fa3a2a19894435b0df6 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/36df617fa6cf4fa3a2a19894435b0df6 2024-12-04T14:58:28,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/094a60d3e78c48fbb4a07456d16aac0e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/094a60d3e78c48fbb4a07456d16aac0e 2024-12-04T14:58:28,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/bb88141640124429867e4114ae48a83b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/bb88141640124429867e4114ae48a83b 2024-12-04T14:58:28,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/939d04133a7a40d79a8ec19cd3b49dcc to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/939d04133a7a40d79a8ec19cd3b49dcc 2024-12-04T14:58:28,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4614de8e33e14b14baba09f5bb516df2 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4614de8e33e14b14baba09f5bb516df2 2024-12-04T14:58:28,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c60fb86da5f94e0385cfcae98a07ceda to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c60fb86da5f94e0385cfcae98a07ceda 2024-12-04T14:58:28,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/473b722441f1417fb81186fa98e4cf65 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/473b722441f1417fb81186fa98e4cf65 2024-12-04T14:58:28,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0eb67333d3ff438bbf8d88d4def9aa01 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0eb67333d3ff438bbf8d88d4def9aa01 2024-12-04T14:58:28,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4b2fc7bd971f494e84ae4dc2fc06e708 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4b2fc7bd971f494e84ae4dc2fc06e708 2024-12-04T14:58:28,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/2d6ab322c8634929b2bbb00c6a4d8c94 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/2d6ab322c8634929b2bbb00c6a4d8c94 2024-12-04T14:58:28,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/e7f9b194c7cb4b62b5b8dfd42cf045e6 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/e7f9b194c7cb4b62b5b8dfd42cf045e6 2024-12-04T14:58:28,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c9fae2753cd74da28113cab30d25827d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/c9fae2753cd74da28113cab30d25827d 2024-12-04T14:58:28,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/1cb2083e229b46b28e188c9ebced8fcc to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/1cb2083e229b46b28e188c9ebced8fcc 2024-12-04T14:58:28,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3e96b4a216184273a8387edb8511f062 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/3e96b4a216184273a8387edb8511f062 2024-12-04T14:58:28,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/283c1e0432eb466a810ebf23304318eb to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/283c1e0432eb466a810ebf23304318eb 2024-12-04T14:58:28,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/81c244c86f3f4a588222afd09a9a58ae to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/81c244c86f3f4a588222afd09a9a58ae 2024-12-04T14:58:28,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ca609af3838a4850838d4c9f34174d53 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ca609af3838a4850838d4c9f34174d53 2024-12-04T14:58:28,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ce149c976dfd49779b4c3707bb5ad64e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/ce149c976dfd49779b4c3707bb5ad64e 2024-12-04T14:58:28,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/16b1eeab787d4541bc6928fb32bd3f2f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/16b1eeab787d4541bc6928fb32bd3f2f 2024-12-04T14:58:28,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/02bea73caca34f249241bd306b89024a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/02bea73caca34f249241bd306b89024a 2024-12-04T14:58:28,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4eea8e286fec429182e52f34a7eb69e7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/4eea8e286fec429182e52f34a7eb69e7 2024-12-04T14:58:28,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0ac505ae0df540c98dd258aef1ec4db1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/0ac505ae0df540c98dd258aef1ec4db1 2024-12-04T14:58:28,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/c997abbab56c4f93847d9fdb1edba618, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5af0224f2dd74578bb53d62323b1417f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/305ba369f4124e098c505dd53429aa68, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/ad9b827d720f4657bd884f4bae91c50a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/94b8bed6fcd7420f9c93228118ede395, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f2d449f814be46a49cd01f765be5fbf5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/198f8b0c989a4558a089d11d60b624bd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/06de4e12eac1414c98537a884cbab90a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e8dd08d212f740aab46fee10f7f62ee7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/cbbe849aeb9940faa105d33460b5f14f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e95cf47ba9f143d2934628f28a769681, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/d095ae610fdb49448ee94bc33e502f87, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/a445c59b4d494f75bc0f76aa0cb9faf7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5303ecb0c8f348d685f0e76348d3bf60, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/fdd7159d8523478ea391accab5ef4f9c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/797dd6d6a6d54f49a6b3c3151fca45fe, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/8a354e8249ae4801ab69c7d0fcd84e03, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e5e20f9589d1461684993d5603b18c07, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/796d6bdf4c614c359c2ef909979e311b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/515c330759784097aaea9caa0fc9556d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/3e38784b91ad49318f5bf0b7a62b5840, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f8b4bce840d9484db0e8a249fc39baee, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/1445eeaf06f0494bbc3d8979debee7c5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/bd38638ff82342aeaa305a9a1b9898c4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/6318b1766eec4b8fb7ebcf017bc095a1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b4679369c4694aea967be3f9db710422, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b2367ee528aa4754a16ba837164ae0a5] to archive 2024-12-04T14:58:28,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:58:28,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/c997abbab56c4f93847d9fdb1edba618 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/c997abbab56c4f93847d9fdb1edba618 2024-12-04T14:58:28,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5af0224f2dd74578bb53d62323b1417f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5af0224f2dd74578bb53d62323b1417f 2024-12-04T14:58:28,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/305ba369f4124e098c505dd53429aa68 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/305ba369f4124e098c505dd53429aa68 2024-12-04T14:58:28,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/ad9b827d720f4657bd884f4bae91c50a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/ad9b827d720f4657bd884f4bae91c50a 2024-12-04T14:58:28,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/94b8bed6fcd7420f9c93228118ede395 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/94b8bed6fcd7420f9c93228118ede395 2024-12-04T14:58:28,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f2d449f814be46a49cd01f765be5fbf5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f2d449f814be46a49cd01f765be5fbf5 2024-12-04T14:58:28,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/198f8b0c989a4558a089d11d60b624bd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/198f8b0c989a4558a089d11d60b624bd 2024-12-04T14:58:28,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/06de4e12eac1414c98537a884cbab90a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/06de4e12eac1414c98537a884cbab90a 2024-12-04T14:58:28,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e8dd08d212f740aab46fee10f7f62ee7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e8dd08d212f740aab46fee10f7f62ee7 2024-12-04T14:58:28,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/cbbe849aeb9940faa105d33460b5f14f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/cbbe849aeb9940faa105d33460b5f14f 2024-12-04T14:58:28,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e95cf47ba9f143d2934628f28a769681 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e95cf47ba9f143d2934628f28a769681 2024-12-04T14:58:28,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/d095ae610fdb49448ee94bc33e502f87 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/d095ae610fdb49448ee94bc33e502f87 2024-12-04T14:58:28,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/a445c59b4d494f75bc0f76aa0cb9faf7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/a445c59b4d494f75bc0f76aa0cb9faf7 2024-12-04T14:58:28,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5303ecb0c8f348d685f0e76348d3bf60 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/5303ecb0c8f348d685f0e76348d3bf60 2024-12-04T14:58:28,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/fdd7159d8523478ea391accab5ef4f9c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/fdd7159d8523478ea391accab5ef4f9c 2024-12-04T14:58:28,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/797dd6d6a6d54f49a6b3c3151fca45fe to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/797dd6d6a6d54f49a6b3c3151fca45fe 2024-12-04T14:58:28,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/8a354e8249ae4801ab69c7d0fcd84e03 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/8a354e8249ae4801ab69c7d0fcd84e03 2024-12-04T14:58:28,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e5e20f9589d1461684993d5603b18c07 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/e5e20f9589d1461684993d5603b18c07 2024-12-04T14:58:28,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/796d6bdf4c614c359c2ef909979e311b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/796d6bdf4c614c359c2ef909979e311b 2024-12-04T14:58:28,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/515c330759784097aaea9caa0fc9556d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/515c330759784097aaea9caa0fc9556d 2024-12-04T14:58:28,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/3e38784b91ad49318f5bf0b7a62b5840 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/3e38784b91ad49318f5bf0b7a62b5840 2024-12-04T14:58:28,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f8b4bce840d9484db0e8a249fc39baee to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f8b4bce840d9484db0e8a249fc39baee 2024-12-04T14:58:28,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/1445eeaf06f0494bbc3d8979debee7c5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/1445eeaf06f0494bbc3d8979debee7c5 2024-12-04T14:58:28,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/bd38638ff82342aeaa305a9a1b9898c4 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/bd38638ff82342aeaa305a9a1b9898c4 2024-12-04T14:58:28,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/6318b1766eec4b8fb7ebcf017bc095a1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/6318b1766eec4b8fb7ebcf017bc095a1 2024-12-04T14:58:28,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b4679369c4694aea967be3f9db710422 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b4679369c4694aea967be3f9db710422 2024-12-04T14:58:28,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b2367ee528aa4754a16ba837164ae0a5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/b2367ee528aa4754a16ba837164ae0a5 2024-12-04T14:58:28,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/732e0482a3ed40d0943bfee816cb2f17, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1de1a0d3c08546feaca749642a9eb9d5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/21e740a42aaa4b04969f0250f4601eaa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/965dc71061e84982bcde94c00afc8f9b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/582d643f367b44e885832ba2fd196cec, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/2d18c426621140dea250ad17177f6bda, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/29278334d4ef471083a3a0098712cdda, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/7dded204e9f94d3e80ae07bc772f4864, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e30eb49c2da34c8595c2e655449f4b38, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/fcec02e77d2f4814b425a61f802a2572, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/48437bec304442b08c4c6788e45c22b8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6949a2379b95427a8a80a610c04d1e1b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/f38b04859b554e6d8c629fc1e342d2a8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4fce5c4175724559a85c728ed8730cab, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e56cfbad5ce4486a897270860c8bd9cf, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/d9092b96d1f04dac96e56e7c00e6694a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c3b91d969fc84807aefacf416a944dc6, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/8ee52e5343d34a1da26406df2db250cd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e94a1e942b994c989fe8b6f8a57fe2df, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c92c08481b57437ca8a44690dab94a3a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e5b196b81b19437aa6a7e3d1115187b1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1b46417b342a4d8cb3c1235199aa237c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/ef130139226746ffaac07cee5d021d28, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/863289ba8a6542e4861edf251e113eeb, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6d0e710ae9bc407f9ea8c8df6eccc25b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4e0e72766cd4497399208af8357a5d6b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/72035906835f4c7bb69b8aca7becef69] to archive 2024-12-04T14:58:28,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:58:28,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/732e0482a3ed40d0943bfee816cb2f17 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/732e0482a3ed40d0943bfee816cb2f17 2024-12-04T14:58:28,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1de1a0d3c08546feaca749642a9eb9d5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1de1a0d3c08546feaca749642a9eb9d5 2024-12-04T14:58:28,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/21e740a42aaa4b04969f0250f4601eaa to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/21e740a42aaa4b04969f0250f4601eaa 2024-12-04T14:58:28,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/965dc71061e84982bcde94c00afc8f9b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/965dc71061e84982bcde94c00afc8f9b 2024-12-04T14:58:28,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/582d643f367b44e885832ba2fd196cec to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/582d643f367b44e885832ba2fd196cec 2024-12-04T14:58:28,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/2d18c426621140dea250ad17177f6bda to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/2d18c426621140dea250ad17177f6bda 2024-12-04T14:58:28,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/29278334d4ef471083a3a0098712cdda to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/29278334d4ef471083a3a0098712cdda 2024-12-04T14:58:28,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/7dded204e9f94d3e80ae07bc772f4864 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/7dded204e9f94d3e80ae07bc772f4864 2024-12-04T14:58:28,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e30eb49c2da34c8595c2e655449f4b38 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e30eb49c2da34c8595c2e655449f4b38 2024-12-04T14:58:28,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/fcec02e77d2f4814b425a61f802a2572 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/fcec02e77d2f4814b425a61f802a2572 2024-12-04T14:58:28,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/48437bec304442b08c4c6788e45c22b8 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/48437bec304442b08c4c6788e45c22b8 2024-12-04T14:58:28,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6949a2379b95427a8a80a610c04d1e1b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6949a2379b95427a8a80a610c04d1e1b 2024-12-04T14:58:28,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/f38b04859b554e6d8c629fc1e342d2a8 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/f38b04859b554e6d8c629fc1e342d2a8 2024-12-04T14:58:28,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4fce5c4175724559a85c728ed8730cab to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4fce5c4175724559a85c728ed8730cab 2024-12-04T14:58:28,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e56cfbad5ce4486a897270860c8bd9cf to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e56cfbad5ce4486a897270860c8bd9cf 2024-12-04T14:58:28,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/d9092b96d1f04dac96e56e7c00e6694a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/d9092b96d1f04dac96e56e7c00e6694a 2024-12-04T14:58:28,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c3b91d969fc84807aefacf416a944dc6 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c3b91d969fc84807aefacf416a944dc6 2024-12-04T14:58:28,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/8ee52e5343d34a1da26406df2db250cd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/8ee52e5343d34a1da26406df2db250cd 2024-12-04T14:58:28,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e94a1e942b994c989fe8b6f8a57fe2df to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e94a1e942b994c989fe8b6f8a57fe2df 2024-12-04T14:58:28,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c92c08481b57437ca8a44690dab94a3a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/c92c08481b57437ca8a44690dab94a3a 2024-12-04T14:58:28,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e5b196b81b19437aa6a7e3d1115187b1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/e5b196b81b19437aa6a7e3d1115187b1 2024-12-04T14:58:28,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1b46417b342a4d8cb3c1235199aa237c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/1b46417b342a4d8cb3c1235199aa237c 2024-12-04T14:58:28,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/ef130139226746ffaac07cee5d021d28 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/ef130139226746ffaac07cee5d021d28 2024-12-04T14:58:28,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/863289ba8a6542e4861edf251e113eeb to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/863289ba8a6542e4861edf251e113eeb 2024-12-04T14:58:28,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6d0e710ae9bc407f9ea8c8df6eccc25b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/6d0e710ae9bc407f9ea8c8df6eccc25b 2024-12-04T14:58:28,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4e0e72766cd4497399208af8357a5d6b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/4e0e72766cd4497399208af8357a5d6b 2024-12-04T14:58:28,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/72035906835f4c7bb69b8aca7becef69 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/72035906835f4c7bb69b8aca7becef69 2024-12-04T14:58:28,933 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/recovered.edits/390.seqid, newMaxSeqId=390, maxSeqId=1 2024-12-04T14:58:28,933 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. 2024-12-04T14:58:28,933 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 031e210d034f5b12ccb5e29a336d9de0: Waiting for close lock at 1733324307607Running coprocessor pre-close hooks at 1733324307607Disabling compacts and flushes for region at 1733324307607Disabling writes for close at 1733324307607Obtaining lock to block concurrent updates at 1733324307608 (+1 ms)Preparing flush snapshotting stores in 031e210d034f5b12ccb5e29a336d9de0 at 1733324307608Finished memstore snapshotting TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1733324307608Flushing stores of TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0. at 1733324307609 (+1 ms)Flushing 031e210d034f5b12ccb5e29a336d9de0/A: creating writer at 1733324307609Flushing 031e210d034f5b12ccb5e29a336d9de0/A: appending metadata at 1733324307614 (+5 ms)Flushing 031e210d034f5b12ccb5e29a336d9de0/A: closing flushed file at 1733324307614Flushing 031e210d034f5b12ccb5e29a336d9de0/B: creating writer at 1733324308029 (+415 ms)Flushing 031e210d034f5b12ccb5e29a336d9de0/B: appending metadata at 1733324308033 (+4 ms)Flushing 031e210d034f5b12ccb5e29a336d9de0/B: closing flushed file at 1733324308033Flushing 031e210d034f5b12ccb5e29a336d9de0/C: creating writer at 1733324308442 (+409 ms)Flushing 031e210d034f5b12ccb5e29a336d9de0/C: appending metadata at 1733324308444 (+2 ms)Flushing 031e210d034f5b12ccb5e29a336d9de0/C: closing flushed file at 1733324308444Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b083da3: reopening flushed file at 1733324308852 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14942c7e: reopening flushed file at 1733324308856 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b971ca2: reopening flushed file at 1733324308860 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 031e210d034f5b12ccb5e29a336d9de0 in 1257ms, sequenceid=387, compaction requested=false at 1733324308864 (+4 ms)Writing region close event to WAL at 1733324308930 (+66 ms)Running coprocessor post-close hooks at 1733324308933 (+3 ms)Closed at 1733324308933 2024-12-04T14:58:28,935 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:28,935 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=031e210d034f5b12ccb5e29a336d9de0, regionState=CLOSED 2024-12-04T14:58:28,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:58:28,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-04T14:58:28,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 031e210d034f5b12ccb5e29a336d9de0, server=d214edae7bd2,34477,1733324158096 in 1.4830 sec 2024-12-04T14:58:28,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-12-04T14:58:28,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=031e210d034f5b12ccb5e29a336d9de0, UNASSIGN in 1.4860 sec 2024-12-04T14:58:28,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-04T14:58:28,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4880 sec 2024-12-04T14:58:28,942 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324308941"}]},"ts":"1733324308941"} 2024-12-04T14:58:28,943 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T14:58:28,943 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T14:58:28,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4950 sec 2024-12-04T14:58:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T14:58:29,596 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:29,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T14:58:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:29,600 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T14:58:29,601 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:29,604 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:29,607 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/recovered.edits] 2024-12-04T14:58:29,610 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/645f2d6923c34b95a87726c87a76bc34 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/645f2d6923c34b95a87726c87a76bc34 2024-12-04T14:58:29,611 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/abf238246ba74bfaa733e9a312318687 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/A/abf238246ba74bfaa733e9a312318687 2024-12-04T14:58:29,613 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/8d167d7915d8481da3b0ea8140811622 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/8d167d7915d8481da3b0ea8140811622 2024-12-04T14:58:29,614 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f95a7615de3c493c829e44ee7667447a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/B/f95a7615de3c493c829e44ee7667447a 2024-12-04T14:58:29,616 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/3ff61f588ad644d9b2f7d3dec3e3cc26 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/3ff61f588ad644d9b2f7d3dec3e3cc26 2024-12-04T14:58:29,616 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/fbd58683cda34bdc95ad159bb31d8dd1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/C/fbd58683cda34bdc95ad159bb31d8dd1 2024-12-04T14:58:29,618 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/recovered.edits/390.seqid to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0/recovered.edits/390.seqid 2024-12-04T14:58:29,618 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/031e210d034f5b12ccb5e29a336d9de0 2024-12-04T14:58:29,618 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T14:58:29,620 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:29,621 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T14:58:29,623 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T14:58:29,624 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:29,625 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T14:58:29,625 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324309625"}]},"ts":"9223372036854775807"} 2024-12-04T14:58:29,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T14:58:29,627 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 031e210d034f5b12ccb5e29a336d9de0, NAME => 'TestAcidGuarantees,,1733324274604.031e210d034f5b12ccb5e29a336d9de0.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T14:58:29,627 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T14:58:29,627 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324309627"}]},"ts":"9223372036854775807"} 2024-12-04T14:58:29,629 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T14:58:29,630 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:29,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-12-04T14:58:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T14:58:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T14:58:29,706 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:29,712 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 242), OpenFileDescriptor=445 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=473 (was 399) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9444 (was 9602) 2024-12-04T14:58:29,719 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=473, ProcessCount=11, AvailableMemoryMB=9443 2024-12-04T14:58:29,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:58:29,721 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:58:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:29,722 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T14:58:29,722 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:29,722 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-12-04T14:58:29,723 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T14:58:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T14:58:29,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742291_1467 (size=959) 2024-12-04T14:58:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T14:58:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T14:58:30,130 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1 2024-12-04T14:58:30,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742292_1468 (size=53) 2024-12-04T14:58:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T14:58:30,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:30,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e727f8587f46eb6f1b53c466f4689322, disabling compactions & flushes 2024-12-04T14:58:30,542 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:30,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:30,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. after waiting 0 ms 2024-12-04T14:58:30,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:30,542 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:30,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e727f8587f46eb6f1b53c466f4689322: Waiting for close lock at 1733324310541Disabling compacts and flushes for region at 1733324310541Disabling writes for close at 1733324310542 (+1 ms)Writing region close event to WAL at 1733324310542Closed at 1733324310542 2024-12-04T14:58:30,544 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T14:58:30,544 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733324310544"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733324310544"}]},"ts":"1733324310544"} 2024-12-04T14:58:30,545 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T14:58:30,546 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T14:58:30,546 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324310546"}]},"ts":"1733324310546"} 2024-12-04T14:58:30,547 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T14:58:30,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, ASSIGN}] 2024-12-04T14:58:30,548 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, ASSIGN 2024-12-04T14:58:30,549 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, ASSIGN; state=OFFLINE, location=d214edae7bd2,34477,1733324158096; forceNewPlan=false, retain=false 2024-12-04T14:58:30,699 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=e727f8587f46eb6f1b53c466f4689322, regionState=OPENING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:30,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, ASSIGN because future has completed 2024-12-04T14:58:30,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:58:30,860 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:30,860 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:58:30,861 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,861 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:30,861 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,861 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,863 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,864 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e727f8587f46eb6f1b53c466f4689322 columnFamilyName A 2024-12-04T14:58:30,864 DEBUG [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:30,864 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(327): Store=e727f8587f46eb6f1b53c466f4689322/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:30,865 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T14:58:30,865 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e727f8587f46eb6f1b53c466f4689322 columnFamilyName B 2024-12-04T14:58:30,866 DEBUG [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:30,866 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(327): Store=e727f8587f46eb6f1b53c466f4689322/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:30,866 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,867 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e727f8587f46eb6f1b53c466f4689322 columnFamilyName C 2024-12-04T14:58:30,867 DEBUG [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:30,867 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(327): Store=e727f8587f46eb6f1b53c466f4689322/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:30,868 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,868 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,868 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,869 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,869 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,869 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:58:30,870 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,872 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T14:58:30,872 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened e727f8587f46eb6f1b53c466f4689322; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62060739, jitterRate=-0.0752229243516922}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:58:30,872 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:30,873 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for e727f8587f46eb6f1b53c466f4689322: Running coprocessor pre-open hook at 1733324310861Writing region info on filesystem at 1733324310861Initializing all the Stores at 1733324310862 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324310862Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324310863 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324310863Cleaning up temporary data from old regions at 1733324310869 (+6 ms)Running coprocessor post-open hooks at 1733324310872 (+3 ms)Region opened successfully at 1733324310873 (+1 ms) 2024-12-04T14:58:30,873 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., pid=138, masterSystemTime=1733324310854 2024-12-04T14:58:30,875 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:30,875 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:30,876 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=e727f8587f46eb6f1b53c466f4689322, regionState=OPEN, openSeqNum=2, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:30,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:58:30,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-04T14:58:30,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 in 176 msec 2024-12-04T14:58:30,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-04T14:58:30,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, ASSIGN in 333 msec 2024-12-04T14:58:30,882 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T14:58:30,883 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324310882"}]},"ts":"1733324310882"} 2024-12-04T14:58:30,884 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T14:58:30,884 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T14:58:30,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-12-04T14:58:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-04T14:58:31,875 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:31,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:31,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:31,879 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1b93a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:31,883 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb8762d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:31,885 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:31,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:31,893 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T14:58:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is d214edae7bd2,45797,1733324157426 2024-12-04T14:58:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7be98fba 2024-12-04T14:58:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T14:58:31,895 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T14:58:31,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T14:58:31,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T14:58:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T14:58:31,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742293_1469 (size=995) 2024-12-04T14:58:31,940 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-04T14:58:31,940 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-04T14:58:31,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:58:31,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, REOPEN/MOVE}] 2024-12-04T14:58:31,955 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, REOPEN/MOVE 2024-12-04T14:58:31,957 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e727f8587f46eb6f1b53c466f4689322, regionState=CLOSING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:31,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, REOPEN/MOVE because future has completed 2024-12-04T14:58:31,962 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:58:31,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:58:32,115 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,115 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:58:32,115 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing e727f8587f46eb6f1b53c466f4689322, disabling compactions & flushes 2024-12-04T14:58:32,115 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,115 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,115 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. after waiting 0 ms 2024-12-04T14:58:32,115 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,120 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T14:58:32,121 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,121 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for e727f8587f46eb6f1b53c466f4689322: Waiting for close lock at 1733324312115Running coprocessor pre-close hooks at 1733324312115Disabling compacts and flushes for region at 1733324312115Disabling writes for close at 1733324312115Writing region close event to WAL at 1733324312116 (+1 ms)Running coprocessor post-close hooks at 1733324312121 (+5 ms)Closed at 1733324312121 2024-12-04T14:58:32,121 WARN [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: e727f8587f46eb6f1b53c466f4689322 to self. 2024-12-04T14:58:32,125 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e727f8587f46eb6f1b53c466f4689322, regionState=CLOSED 2024-12-04T14:58:32,126 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:58:32,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-04T14:58:32,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 in 169 msec 2024-12-04T14:58:32,135 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, REOPEN/MOVE; state=CLOSED, location=d214edae7bd2,34477,1733324158096; forceNewPlan=false, retain=true 2024-12-04T14:58:32,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e727f8587f46eb6f1b53c466f4689322, regionState=OPENING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, REOPEN/MOVE because future has completed 2024-12-04T14:58:32,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:58:32,443 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,444 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} 2024-12-04T14:58:32,444 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,444 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T14:58:32,444 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,444 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,446 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,447 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e727f8587f46eb6f1b53c466f4689322 columnFamilyName A 2024-12-04T14:58:32,448 DEBUG [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:32,449 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(327): Store=e727f8587f46eb6f1b53c466f4689322/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:32,449 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,450 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e727f8587f46eb6f1b53c466f4689322 columnFamilyName B 2024-12-04T14:58:32,450 DEBUG [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:32,450 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(327): Store=e727f8587f46eb6f1b53c466f4689322/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:32,450 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,451 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e727f8587f46eb6f1b53c466f4689322 columnFamilyName C 2024-12-04T14:58:32,451 DEBUG [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:32,451 INFO [StoreOpener-e727f8587f46eb6f1b53c466f4689322-1 {}] regionserver.HStore(327): Store=e727f8587f46eb6f1b53c466f4689322/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T14:58:32,452 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,452 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,454 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,454 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,454 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,455 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T14:58:32,456 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,456 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened e727f8587f46eb6f1b53c466f4689322; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64431589, jitterRate=-0.039894506335258484}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T14:58:32,456 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,457 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for e727f8587f46eb6f1b53c466f4689322: Running coprocessor pre-open hook at 1733324312444Writing region info on filesystem at 1733324312444Initializing all the Stores at 1733324312446 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324312446Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324312446Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324312446Cleaning up temporary data from old regions at 1733324312454 (+8 ms)Running coprocessor post-open hooks at 1733324312456 (+2 ms)Region opened successfully at 1733324312457 (+1 ms) 2024-12-04T14:58:32,457 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., pid=143, masterSystemTime=1733324312441 2024-12-04T14:58:32,459 DEBUG [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,459 INFO [RS_OPEN_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,459 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e727f8587f46eb6f1b53c466f4689322, regionState=OPEN, openSeqNum=5, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:58:32,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-12-04T14:58:32,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 in 173 msec 2024-12-04T14:58:32,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-04T14:58:32,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, REOPEN/MOVE in 509 msec 2024-12-04T14:58:32,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-04T14:58:32,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-12-04T14:58:32,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 588 msec 2024-12-04T14:58:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-04T14:58:32,488 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:32,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:32,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:32,491 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219c8ace, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:32,492 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d85cf59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:32,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:32,495 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d99ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:32,497 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aefb29b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:32,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:32,502 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3baf0253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:32,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:32,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:32,504 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5022611e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:32,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:32,507 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ef6f4f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:32,508 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@368afcdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:32,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:32,513 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3daae4c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:32,515 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b303f56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:32,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:32,520 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b3c26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:32,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:32,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:32,522 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:32,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2ea993, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,527 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:32,527 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:32,528 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f470959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:32,531 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ee86d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:32,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:32,534 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:32,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c2c646b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:32,537 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:32,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67846523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:32,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:32,540 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705cab2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:32,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:32,542 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b754da8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d214edae7bd2,45797,-1 for getting cluster id 2024-12-04T14:58:32,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T14:58:32,544 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1d58f65-ea39-40cd-ad65-dc760c49aa67' 2024-12-04T14:58:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T14:58:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1d58f65-ea39-40cd-ad65-dc760c49aa67" 2024-12-04T14:58:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a9a2693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d214edae7bd2,45797,-1] 2024-12-04T14:58:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T14:58:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:32,546 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T14:58:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a1f651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T14:58:32,549 DEBUG [Thread-2138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:32,550 DEBUG [Thread-2142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:32,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:58:32,555 DEBUG [Thread-2136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:32,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:32,557 DEBUG [Thread-2127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:32,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:58:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:58:32,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:32,560 DEBUG [Thread-2133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:32,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:58:32,561 DEBUG [Thread-2133 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:32,561 DEBUG [Thread-2133 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:32,561 DEBUG [Thread-2131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:32,562 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:32,562 DEBUG [Thread-2131 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:32,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:32,563 DEBUG [Thread-2131 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-04T14:58:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T14:58:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:58:32,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:32,566 DEBUG [Thread-2140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:32,566 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:58:32,567 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:32,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:32,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:58:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,575 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:32,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:58:32,575 DEBUG [Thread-2125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:32,580 DEBUG [Thread-2129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:32,580 DEBUG [Thread-2144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T14:58:32,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:32,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:32,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:32,582 DEBUG [Thread-2144 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d214edae7bd2,34477,1733324158096, seqNum=-1] 2024-12-04T14:58:32,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:32,583 DEBUG [Thread-2144 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T14:58:32,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T14:58:32,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:58:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:58:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5] 2024-12-04T14:58:32,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b91ab7c5d19a4dd59e4873410977ccd9_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324312575/Put/seqid=0 2024-12-04T14:58:32,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34628 deadline: 1733324372639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34654 deadline: 1733324372642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324372649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742294_1470 (size=12154) 2024-12-04T14:58:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34636 deadline: 1733324372649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,658 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:32,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34664 deadline: 1733324372656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,663 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b91ab7c5d19a4dd59e4873410977ccd9_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b91ab7c5d19a4dd59e4873410977ccd9_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:32,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/737551c1feb7488ea352b16c9c0166fe, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:32,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/737551c1feb7488ea352b16c9c0166fe is 175, key is test_row_0/A:col10/1733324312575/Put/seqid=0 2024-12-04T14:58:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T14:58:32,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742295_1471 (size=30955) 2024-12-04T14:58:32,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/737551c1feb7488ea352b16c9c0166fe 2024-12-04T14:58:32,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:32,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:32,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b6d94b65d83545efa856c01facc2d9bd is 50, key is test_row_0/B:col10/1733324312575/Put/seqid=0 2024-12-04T14:58:32,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34636 deadline: 1733324372765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34628 deadline: 1733324372764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34664 deadline: 1733324372765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34654 deadline: 1733324372765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324372765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742296_1472 (size=12001) 2024-12-04T14:58:32,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b6d94b65d83545efa856c01facc2d9bd 2024-12-04T14:58:32,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/2b7ff23bcb6a4d319b9b7ed98f7932bb is 50, key is test_row_0/C:col10/1733324312575/Put/seqid=0 2024-12-04T14:58:32,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742297_1473 (size=12001) 2024-12-04T14:58:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T14:58:32,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:32,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:32,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34636 deadline: 1733324372975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34628 deadline: 1733324372975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34654 deadline: 1733324372975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34664 deadline: 1733324372975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:32,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324372985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:33,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:33,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T14:58:33,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:33,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:33,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/2b7ff23bcb6a4d319b9b7ed98f7932bb 2024-12-04T14:58:33,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/737551c1feb7488ea352b16c9c0166fe as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/737551c1feb7488ea352b16c9c0166fe 2024-12-04T14:58:33,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/737551c1feb7488ea352b16c9c0166fe, entries=150, sequenceid=16, filesize=30.2 K 2024-12-04T14:58:33,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b6d94b65d83545efa856c01facc2d9bd as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b6d94b65d83545efa856c01facc2d9bd 2024-12-04T14:58:33,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b6d94b65d83545efa856c01facc2d9bd, entries=150, sequenceid=16, filesize=11.7 K 2024-12-04T14:58:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34636 deadline: 1733324373286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/2b7ff23bcb6a4d319b9b7ed98f7932bb as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2b7ff23bcb6a4d319b9b7ed98f7932bb 2024-12-04T14:58:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34628 deadline: 1733324373286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34664 deadline: 1733324373286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34654 deadline: 1733324373286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2b7ff23bcb6a4d319b9b7ed98f7932bb, entries=150, sequenceid=16, filesize=11.7 K 2024-12-04T14:58:33,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e727f8587f46eb6f1b53c466f4689322 in 720ms, sequenceid=16, compaction requested=false 2024-12-04T14:58:33,295 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T14:58:33,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:33,303 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:33,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:58:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:33,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047e779856a8b24c3cb7d6f92a9f3329f2_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324312644/Put/seqid=0 2024-12-04T14:58:33,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324373395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742298_1474 (size=12154) 2024-12-04T14:58:33,417 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:33,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047e779856a8b24c3cb7d6f92a9f3329f2_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047e779856a8b24c3cb7d6f92a9f3329f2_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:33,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/be3eebe54f854b1eb96fd1b08605dc38, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:33,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/be3eebe54f854b1eb96fd1b08605dc38 is 175, key is test_row_0/A:col10/1733324312644/Put/seqid=0 2024-12-04T14:58:33,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742299_1475 (size=30955) 2024-12-04T14:58:33,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:33,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324373505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:33,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:33,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T14:58:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324373715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34654 deadline: 1733324373796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34636 deadline: 1733324373796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34628 deadline: 1733324373796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34664 deadline: 1733324373805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:33,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:33,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:33,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,883 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/be3eebe54f854b1eb96fd1b08605dc38 2024-12-04T14:58:33,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b78a8d52f3284cc397cd8567900c66af is 50, key is test_row_0/B:col10/1733324312644/Put/seqid=0 2024-12-04T14:58:33,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742300_1476 (size=12001) 2024-12-04T14:58:33,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:33,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:33,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:33,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:34,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324374035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:34,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:34,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:34,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:34,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:34,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:34,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:34,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:34,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:34,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:34,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b78a8d52f3284cc397cd8567900c66af 2024-12-04T14:58:34,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/9d1ca03c26b841749c8e4a1b9f767ebe is 50, key is test_row_0/C:col10/1733324312644/Put/seqid=0 2024-12-04T14:58:34,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742301_1477 (size=12001) 2024-12-04T14:58:34,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/9d1ca03c26b841749c8e4a1b9f767ebe 2024-12-04T14:58:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/be3eebe54f854b1eb96fd1b08605dc38 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/be3eebe54f854b1eb96fd1b08605dc38 2024-12-04T14:58:34,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/be3eebe54f854b1eb96fd1b08605dc38, entries=150, sequenceid=41, filesize=30.2 K 2024-12-04T14:58:34,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b78a8d52f3284cc397cd8567900c66af as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b78a8d52f3284cc397cd8567900c66af 2024-12-04T14:58:34,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b78a8d52f3284cc397cd8567900c66af, entries=150, sequenceid=41, filesize=11.7 K 2024-12-04T14:58:34,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/9d1ca03c26b841749c8e4a1b9f767ebe as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/9d1ca03c26b841749c8e4a1b9f767ebe 2024-12-04T14:58:34,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/9d1ca03c26b841749c8e4a1b9f767ebe, entries=150, sequenceid=41, filesize=11.7 K 2024-12-04T14:58:34,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e727f8587f46eb6f1b53c466f4689322 in 1152ms, sequenceid=41, compaction requested=false 2024-12-04T14:58:34,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:34,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-04T14:58:34,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:34,458 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:58:34,470 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T14:58:34,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204112b3121a6d64377a177982ea1ca10c3_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324313372/Put/seqid=0 2024-12-04T14:58:34,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742302_1478 (size=12154) 2024-12-04T14:58:34,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T14:58:34,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324374713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:34,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34654 deadline: 1733324374805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:34,810 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:34,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34636 deadline: 1733324374815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:34,817 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:34,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:34,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34628 deadline: 1733324374815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:34,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34664 deadline: 1733324374815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:34,819 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:34,820 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:34,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324374825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:34,929 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204112b3121a6d64377a177982ea1ca10c3_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204112b3121a6d64377a177982ea1ca10c3_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/7bf55ef5d0604b27bab22000cf1f8ab5, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/7bf55ef5d0604b27bab22000cf1f8ab5 is 175, key is test_row_0/A:col10/1733324313372/Put/seqid=0 2024-12-04T14:58:34,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742303_1479 (size=30955) 2024-12-04T14:58:34,966 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/7bf55ef5d0604b27bab22000cf1f8ab5 2024-12-04T14:58:34,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/a881e8f188294e7d884c45ab99832cdc is 50, key is test_row_0/B:col10/1733324313372/Put/seqid=0 2024-12-04T14:58:35,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742304_1480 (size=12001) 2024-12-04T14:58:35,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324375035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:35,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324375345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:35,437 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/a881e8f188294e7d884c45ab99832cdc 2024-12-04T14:58:35,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/e616b3fb3e204d01beb2cfa76dbb37df is 50, key is test_row_0/C:col10/1733324313372/Put/seqid=0 2024-12-04T14:58:35,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742305_1481 (size=12001) 2024-12-04T14:58:35,494 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/e616b3fb3e204d01beb2cfa76dbb37df 2024-12-04T14:58:35,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/7bf55ef5d0604b27bab22000cf1f8ab5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7bf55ef5d0604b27bab22000cf1f8ab5 2024-12-04T14:58:35,511 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7bf55ef5d0604b27bab22000cf1f8ab5, entries=150, sequenceid=52, filesize=30.2 K 2024-12-04T14:58:35,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/a881e8f188294e7d884c45ab99832cdc as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/a881e8f188294e7d884c45ab99832cdc 2024-12-04T14:58:35,519 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/a881e8f188294e7d884c45ab99832cdc, entries=150, sequenceid=52, filesize=11.7 K 2024-12-04T14:58:35,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/e616b3fb3e204d01beb2cfa76dbb37df as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/e616b3fb3e204d01beb2cfa76dbb37df 2024-12-04T14:58:35,527 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/e616b3fb3e204d01beb2cfa76dbb37df, entries=150, sequenceid=52, filesize=11.7 K 2024-12-04T14:58:35,529 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e727f8587f46eb6f1b53c466f4689322 in 1070ms, sequenceid=52, compaction requested=true 2024-12-04T14:58:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-04T14:58:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-04T14:58:35,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-04T14:58:35,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9630 sec 2024-12-04T14:58:35,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.9710 sec 2024-12-04T14:58:35,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:35,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-04T14:58:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:35,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120458e1527ab0664b1380445573b2052e32_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324315856/Put/seqid=0 2024-12-04T14:58:35,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742306_1482 (size=12154) 2024-12-04T14:58:35,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324375914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:35,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:36,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324376039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:36,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324376245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:36,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:36,319 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:36,323 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120458e1527ab0664b1380445573b2052e32_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120458e1527ab0664b1380445573b2052e32_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:36,326 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0604a6042b6048d8b1723914c0ebb599, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:36,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0604a6042b6048d8b1723914c0ebb599 is 175, key is test_row_0/A:col10/1733324315856/Put/seqid=0 2024-12-04T14:58:36,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742307_1483 (size=30955) 2024-12-04T14:58:36,354 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0604a6042b6048d8b1723914c0ebb599 2024-12-04T14:58:36,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b33b6be5d9a4460d8aeed43ad06b718a is 50, key is test_row_0/B:col10/1733324315856/Put/seqid=0 2024-12-04T14:58:36,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742308_1484 (size=12001) 2024-12-04T14:58:36,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324376555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-04T14:58:36,726 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:36,726 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-04T14:58:36,728 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T14:58:36,731 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:36,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:36,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b33b6be5d9a4460d8aeed43ad06b718a 2024-12-04T14:58:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34654 deadline: 1733324376825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:36,835 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T14:58:36,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34628 deadline: 1733324376835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:36,840 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:36,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:36,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/31f052cd28e141e4983f1300d5e2f17d is 50, key is test_row_0/C:col10/1733324315856/Put/seqid=0 2024-12-04T14:58:36,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34636 deadline: 1733324376845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34664 deadline: 1733324376845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:36,851 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:36,851 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:36,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742309_1485 (size=12001) 2024-12-04T14:58:36,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/31f052cd28e141e4983f1300d5e2f17d 2024-12-04T14:58:36,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T14:58:36,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:36,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:36,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:36,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:36,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0604a6042b6048d8b1723914c0ebb599 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0604a6042b6048d8b1723914c0ebb599 2024-12-04T14:58:36,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0604a6042b6048d8b1723914c0ebb599, entries=150, sequenceid=78, filesize=30.2 K 2024-12-04T14:58:36,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b33b6be5d9a4460d8aeed43ad06b718a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b33b6be5d9a4460d8aeed43ad06b718a 2024-12-04T14:58:36,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b33b6be5d9a4460d8aeed43ad06b718a, entries=150, sequenceid=78, filesize=11.7 K 2024-12-04T14:58:36,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/31f052cd28e141e4983f1300d5e2f17d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/31f052cd28e141e4983f1300d5e2f17d 2024-12-04T14:58:36,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/31f052cd28e141e4983f1300d5e2f17d, entries=150, sequenceid=78, filesize=11.7 K 2024-12-04T14:58:36,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e727f8587f46eb6f1b53c466f4689322 in 1053ms, sequenceid=78, compaction requested=true 2024-12-04T14:58:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:36,917 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:36,917 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T14:58:36,920 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:36,920 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/B is initiating minor compaction (all files) 2024-12-04T14:58:36,920 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/B in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:36,920 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b6d94b65d83545efa856c01facc2d9bd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b78a8d52f3284cc397cd8567900c66af, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/a881e8f188294e7d884c45ab99832cdc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b33b6be5d9a4460d8aeed43ad06b718a] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=46.9 K 2024-12-04T14:58:36,920 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:36,920 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/A is initiating minor compaction (all files) 2024-12-04T14:58:36,920 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/A in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:36,920 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/737551c1feb7488ea352b16c9c0166fe, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/be3eebe54f854b1eb96fd1b08605dc38, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7bf55ef5d0604b27bab22000cf1f8ab5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0604a6042b6048d8b1723914c0ebb599] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=120.9 K 2024-12-04T14:58:36,920 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:36,921 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/737551c1feb7488ea352b16c9c0166fe, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/be3eebe54f854b1eb96fd1b08605dc38, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7bf55ef5d0604b27bab22000cf1f8ab5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0604a6042b6048d8b1723914c0ebb599] 2024-12-04T14:58:36,921 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 737551c1feb7488ea352b16c9c0166fe, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324312571 2024-12-04T14:58:36,921 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b6d94b65d83545efa856c01facc2d9bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324312571 2024-12-04T14:58:36,922 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting be3eebe54f854b1eb96fd1b08605dc38, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733324312638 2024-12-04T14:58:36,922 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b78a8d52f3284cc397cd8567900c66af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733324312638 2024-12-04T14:58:36,922 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bf55ef5d0604b27bab22000cf1f8ab5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733324313342 2024-12-04T14:58:36,922 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting a881e8f188294e7d884c45ab99832cdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733324313342 2024-12-04T14:58:36,923 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0604a6042b6048d8b1723914c0ebb599, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733324314673 2024-12-04T14:58:36,923 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b33b6be5d9a4460d8aeed43ad06b718a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733324314673 2024-12-04T14:58:36,942 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#B#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:36,942 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/6546f881276447868fc5ed469882253e is 50, key is test_row_0/B:col10/1733324315856/Put/seqid=0 2024-12-04T14:58:36,945 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:36,954 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204421d85c5e4e14f4b8231c226366ace03_e727f8587f46eb6f1b53c466f4689322 store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:36,956 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204421d85c5e4e14f4b8231c226366ace03_e727f8587f46eb6f1b53c466f4689322, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:36,956 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204421d85c5e4e14f4b8231c226366ace03_e727f8587f46eb6f1b53c466f4689322 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:36,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742310_1486 (size=12139) 2024-12-04T14:58:36,967 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/6546f881276447868fc5ed469882253e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/6546f881276447868fc5ed469882253e 2024-12-04T14:58:36,975 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/B of e727f8587f46eb6f1b53c466f4689322 into 6546f881276447868fc5ed469882253e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:36,975 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:36,975 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/B, priority=12, startTime=1733324316917; duration=0sec 2024-12-04T14:58:36,975 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:36,975 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:B 2024-12-04T14:58:36,976 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:36,977 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:36,977 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/C is initiating minor compaction (all files) 2024-12-04T14:58:36,977 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/C in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:36,977 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2b7ff23bcb6a4d319b9b7ed98f7932bb, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/9d1ca03c26b841749c8e4a1b9f767ebe, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/e616b3fb3e204d01beb2cfa76dbb37df, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/31f052cd28e141e4983f1300d5e2f17d] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=46.9 K 2024-12-04T14:58:36,977 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b7ff23bcb6a4d319b9b7ed98f7932bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733324312571 2024-12-04T14:58:36,978 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d1ca03c26b841749c8e4a1b9f767ebe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733324312638 2024-12-04T14:58:36,978 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e616b3fb3e204d01beb2cfa76dbb37df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733324313342 2024-12-04T14:58:36,978 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f052cd28e141e4983f1300d5e2f17d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733324314673 2024-12-04T14:58:36,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742311_1487 (size=4469) 2024-12-04T14:58:36,994 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#A#compaction#415 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:36,994 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/d673e433dfa54122bd388825b58f2597 is 175, key is test_row_0/A:col10/1733324315856/Put/seqid=0 2024-12-04T14:58:37,015 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:37,016 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/5aab64e274bf4d45a4a087e08f7fa881 is 50, key is test_row_0/C:col10/1733324315856/Put/seqid=0 2024-12-04T14:58:37,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T14:58:37,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:37,037 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-04T14:58:37,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742312_1488 (size=31093) 2024-12-04T14:58:37,048 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/d673e433dfa54122bd388825b58f2597 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d673e433dfa54122bd388825b58f2597 2024-12-04T14:58:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T14:58:37,054 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/A of e727f8587f46eb6f1b53c466f4689322 into d673e433dfa54122bd388825b58f2597(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:37,054 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:37,055 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/A, priority=12, startTime=1733324316917; duration=0sec 2024-12-04T14:58:37,055 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:37,055 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:A 2024-12-04T14:58:37,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742313_1489 (size=12139) 2024-12-04T14:58:37,068 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/5aab64e274bf4d45a4a087e08f7fa881 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/5aab64e274bf4d45a4a087e08f7fa881 2024-12-04T14:58:37,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:37,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120439285baa41d24ee6b40e05b3dc8e326b_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324315911/Put/seqid=0 2024-12-04T14:58:37,081 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/C of e727f8587f46eb6f1b53c466f4689322 into 5aab64e274bf4d45a4a087e08f7fa881(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:37,081 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:37,081 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/C, priority=12, startTime=1733324316917; duration=0sec 2024-12-04T14:58:37,081 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:37,081 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:C 2024-12-04T14:58:37,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742314_1490 (size=12154) 2024-12-04T14:58:37,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324377233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:37,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324377346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T14:58:37,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:37,510 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120439285baa41d24ee6b40e05b3dc8e326b_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120439285baa41d24ee6b40e05b3dc8e326b_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/d79da3cac2bf47a688d73136ff9e9b25, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:37,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/d79da3cac2bf47a688d73136ff9e9b25 is 175, key is test_row_0/A:col10/1733324315911/Put/seqid=0 2024-12-04T14:58:37,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742315_1491 (size=30955) 2024-12-04T14:58:37,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324377557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T14:58:37,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324377867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:37,925 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/d79da3cac2bf47a688d73136ff9e9b25 2024-12-04T14:58:37,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/401302e0fc734fe9a39187aa49d2a96d is 50, key is test_row_0/B:col10/1733324315911/Put/seqid=0 2024-12-04T14:58:37,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742316_1492 (size=12001) 2024-12-04T14:58:38,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324378375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:38,392 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/401302e0fc734fe9a39187aa49d2a96d 2024-12-04T14:58:38,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/2696d07bb2dc457d8a24b0008c5652c1 is 50, key is test_row_0/C:col10/1733324315911/Put/seqid=0 2024-12-04T14:58:38,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742317_1493 (size=12001) 2024-12-04T14:58:38,833 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/2696d07bb2dc457d8a24b0008c5652c1 2024-12-04T14:58:38,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/d79da3cac2bf47a688d73136ff9e9b25 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d79da3cac2bf47a688d73136ff9e9b25 2024-12-04T14:58:38,847 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d79da3cac2bf47a688d73136ff9e9b25, entries=150, sequenceid=89, filesize=30.2 K 2024-12-04T14:58:38,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/401302e0fc734fe9a39187aa49d2a96d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/401302e0fc734fe9a39187aa49d2a96d 2024-12-04T14:58:38,852 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/401302e0fc734fe9a39187aa49d2a96d, entries=150, sequenceid=89, filesize=11.7 K 2024-12-04T14:58:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/2696d07bb2dc457d8a24b0008c5652c1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2696d07bb2dc457d8a24b0008c5652c1 2024-12-04T14:58:38,861 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2696d07bb2dc457d8a24b0008c5652c1, entries=150, sequenceid=89, filesize=11.7 K 2024-12-04T14:58:38,862 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e727f8587f46eb6f1b53c466f4689322 in 1824ms, sequenceid=89, compaction requested=false 2024-12-04T14:58:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-04T14:58:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-04T14:58:38,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-04T14:58:38,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1320 sec 2024-12-04T14:58:38,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.1390 sec 2024-12-04T14:58:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T14:58:38,885 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:38,886 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-04T14:58:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T14:58:38,888 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:38,889 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:38,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T14:58:39,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T14:58:39,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:39,048 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-04T14:58:39,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204525dd10927c54c68b88c93899e99e33f_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324317225/Put/seqid=0 2024-12-04T14:58:39,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742318_1494 (size=12154) 2024-12-04T14:58:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T14:58:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:39,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:39,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324379428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:39,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T14:58:39,516 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204525dd10927c54c68b88c93899e99e33f_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204525dd10927c54c68b88c93899e99e33f_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:39,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/96578f97103948718bdf560a9b839301, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:39,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/96578f97103948718bdf560a9b839301 is 175, key is test_row_0/A:col10/1733324317225/Put/seqid=0 2024-12-04T14:58:39,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324379535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:39,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:39,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742319_1495 (size=30955) 2024-12-04T14:58:39,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324379745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:39,956 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/96578f97103948718bdf560a9b839301 2024-12-04T14:58:39,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/14ec43d00a044fd98fa6272bde24e07b is 50, key is test_row_0/B:col10/1733324317225/Put/seqid=0 2024-12-04T14:58:39,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742320_1496 (size=12001) 2024-12-04T14:58:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T14:58:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324380055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:40,393 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/14ec43d00a044fd98fa6272bde24e07b 2024-12-04T14:58:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/6d2352b3556049c081f90b21f3d1cdd1 is 50, key is test_row_0/C:col10/1733324317225/Put/seqid=0 2024-12-04T14:58:40,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742321_1497 (size=12001) 2024-12-04T14:58:40,431 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/6d2352b3556049c081f90b21f3d1cdd1 2024-12-04T14:58:40,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/96578f97103948718bdf560a9b839301 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/96578f97103948718bdf560a9b839301 2024-12-04T14:58:40,448 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/96578f97103948718bdf560a9b839301, entries=150, sequenceid=117, filesize=30.2 K 2024-12-04T14:58:40,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/14ec43d00a044fd98fa6272bde24e07b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/14ec43d00a044fd98fa6272bde24e07b 2024-12-04T14:58:40,453 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/14ec43d00a044fd98fa6272bde24e07b, entries=150, sequenceid=117, filesize=11.7 K 2024-12-04T14:58:40,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/6d2352b3556049c081f90b21f3d1cdd1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/6d2352b3556049c081f90b21f3d1cdd1 2024-12-04T14:58:40,458 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/6d2352b3556049c081f90b21f3d1cdd1, entries=150, sequenceid=117, filesize=11.7 K 2024-12-04T14:58:40,459 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e727f8587f46eb6f1b53c466f4689322 in 1411ms, sequenceid=117, compaction requested=true 2024-12-04T14:58:40,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:40,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:40,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-04T14:58:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-04T14:58:40,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-04T14:58:40,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5710 sec 2024-12-04T14:58:40,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.5780 sec 2024-12-04T14:58:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:40,568 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:40,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:58:40,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042889e3e5b2244c5c888ce6ce002ffbff_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324319426/Put/seqid=0 2024-12-04T14:58:40,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742322_1498 (size=12154) 2024-12-04T14:58:40,635 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:40,643 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042889e3e5b2244c5c888ce6ce002ffbff_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042889e3e5b2244c5c888ce6ce002ffbff_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:40,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cb52c3bfe63548dab5389def63988e44, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:40,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cb52c3bfe63548dab5389def63988e44 is 175, key is test_row_0/A:col10/1733324319426/Put/seqid=0 2024-12-04T14:58:40,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742323_1499 (size=30955) 2024-12-04T14:58:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324380716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:40,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324380825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:40,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:40,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34654 deadline: 1733324380855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:40,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34628 deadline: 1733324380855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:40,858 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:40,858 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:40,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34636 deadline: 1733324380885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:40,890 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:40,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34664 deadline: 1733324380895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:40,898 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T14:58:41,036 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:41,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-04T14:58:41,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324381035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:41,039 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:41,040 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:41,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T14:58:41,072 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cb52c3bfe63548dab5389def63988e44 2024-12-04T14:58:41,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/08c943cef3224a1d823b7cda8c22d2ac is 50, key is test_row_0/B:col10/1733324319426/Put/seqid=0 2024-12-04T14:58:41,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742324_1500 (size=12001) 2024-12-04T14:58:41,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/08c943cef3224a1d823b7cda8c22d2ac 2024-12-04T14:58:41,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/eec8137bba69489483ee27ff6fceaec2 is 50, key is test_row_0/C:col10/1733324319426/Put/seqid=0 2024-12-04T14:58:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T14:58:41,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742325_1501 (size=12001) 2024-12-04T14:58:41,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/eec8137bba69489483ee27ff6fceaec2 2024-12-04T14:58:41,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cb52c3bfe63548dab5389def63988e44 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cb52c3bfe63548dab5389def63988e44 2024-12-04T14:58:41,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cb52c3bfe63548dab5389def63988e44, entries=150, sequenceid=128, filesize=30.2 K 2024-12-04T14:58:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/08c943cef3224a1d823b7cda8c22d2ac as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/08c943cef3224a1d823b7cda8c22d2ac 2024-12-04T14:58:41,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/08c943cef3224a1d823b7cda8c22d2ac, entries=150, sequenceid=128, filesize=11.7 K 2024-12-04T14:58:41,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/eec8137bba69489483ee27ff6fceaec2 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/eec8137bba69489483ee27ff6fceaec2 2024-12-04T14:58:41,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/eec8137bba69489483ee27ff6fceaec2, entries=150, sequenceid=128, filesize=11.7 K 2024-12-04T14:58:41,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e727f8587f46eb6f1b53c466f4689322 in 608ms, sequenceid=128, compaction requested=true 2024-12-04T14:58:41,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:41,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:41,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:41,176 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:41,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:41,176 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:41,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:41,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:41,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:41,178 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:41,178 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/B is initiating minor compaction (all files) 2024-12-04T14:58:41,178 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/B in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:41,178 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/6546f881276447868fc5ed469882253e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/401302e0fc734fe9a39187aa49d2a96d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/14ec43d00a044fd98fa6272bde24e07b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/08c943cef3224a1d823b7cda8c22d2ac] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=47.0 K 2024-12-04T14:58:41,180 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123958 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:41,180 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/A is initiating minor compaction (all files) 2024-12-04T14:58:41,180 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/A in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:41,180 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d673e433dfa54122bd388825b58f2597, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d79da3cac2bf47a688d73136ff9e9b25, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/96578f97103948718bdf560a9b839301, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cb52c3bfe63548dab5389def63988e44] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=121.1 K 2024-12-04T14:58:41,180 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:41,180 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d673e433dfa54122bd388825b58f2597, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d79da3cac2bf47a688d73136ff9e9b25, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/96578f97103948718bdf560a9b839301, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cb52c3bfe63548dab5389def63988e44] 2024-12-04T14:58:41,180 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6546f881276447868fc5ed469882253e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733324314673 2024-12-04T14:58:41,181 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting d673e433dfa54122bd388825b58f2597, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733324314673 2024-12-04T14:58:41,181 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 401302e0fc734fe9a39187aa49d2a96d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733324315893 2024-12-04T14:58:41,182 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting d79da3cac2bf47a688d73136ff9e9b25, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733324315893 2024-12-04T14:58:41,182 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14ec43d00a044fd98fa6272bde24e07b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324317190 2024-12-04T14:58:41,182 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 96578f97103948718bdf560a9b839301, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324317190 2024-12-04T14:58:41,183 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08c943cef3224a1d823b7cda8c22d2ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733324319421 2024-12-04T14:58:41,183 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting cb52c3bfe63548dab5389def63988e44, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733324319421 2024-12-04T14:58:41,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T14:58:41,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:41,194 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:58:41,200 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:41,200 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/8261d2db54614a9baa7a527db8e1251f is 50, key is test_row_0/B:col10/1733324319426/Put/seqid=0 2024-12-04T14:58:41,216 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:41,239 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204a9e0efd65e3c4f5daca7b7f72d234411_e727f8587f46eb6f1b53c466f4689322 store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:41,240 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204a9e0efd65e3c4f5daca7b7f72d234411_e727f8587f46eb6f1b53c466f4689322, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:41,240 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a9e0efd65e3c4f5daca7b7f72d234411_e727f8587f46eb6f1b53c466f4689322 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:41,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120458d0856d05084de0aaa70172a8ff6a3a_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324320714/Put/seqid=0 2024-12-04T14:58:41,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742326_1502 (size=12275) 2024-12-04T14:58:41,272 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/8261d2db54614a9baa7a527db8e1251f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8261d2db54614a9baa7a527db8e1251f 2024-12-04T14:58:41,276 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/B of e727f8587f46eb6f1b53c466f4689322 into 8261d2db54614a9baa7a527db8e1251f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:41,276 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:41,276 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/B, priority=12, startTime=1733324321176; duration=0sec 2024-12-04T14:58:41,277 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:41,277 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:B 2024-12-04T14:58:41,277 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:41,278 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:41,278 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/C is initiating minor compaction (all files) 2024-12-04T14:58:41,278 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/C in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:41,278 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/5aab64e274bf4d45a4a087e08f7fa881, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2696d07bb2dc457d8a24b0008c5652c1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/6d2352b3556049c081f90b21f3d1cdd1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/eec8137bba69489483ee27ff6fceaec2] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=47.0 K 2024-12-04T14:58:41,278 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aab64e274bf4d45a4a087e08f7fa881, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733324314673 2024-12-04T14:58:41,279 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2696d07bb2dc457d8a24b0008c5652c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733324315893 2024-12-04T14:58:41,279 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d2352b3556049c081f90b21f3d1cdd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733324317190 2024-12-04T14:58:41,279 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting eec8137bba69489483ee27ff6fceaec2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733324319421 2024-12-04T14:58:41,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742327_1503 (size=4469) 2024-12-04T14:58:41,325 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#A#compaction#427 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:41,325 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cbbafac124cb405994f73468217d2d80 is 175, key is test_row_0/A:col10/1733324319426/Put/seqid=0 2024-12-04T14:58:41,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742328_1504 (size=12304) 2024-12-04T14:58:41,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,340 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120458d0856d05084de0aaa70172a8ff6a3a_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120458d0856d05084de0aaa70172a8ff6a3a_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:41,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cfa5c427ddc14b50b3ffba9e488db742, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:41,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cfa5c427ddc14b50b3ffba9e488db742 is 175, key is test_row_0/A:col10/1733324320714/Put/seqid=0 2024-12-04T14:58:41,350 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:41,351 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/7027cc679b8d4d9b882e8faa436d571a is 50, key is test_row_0/C:col10/1733324319426/Put/seqid=0 2024-12-04T14:58:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:41,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T14:58:41,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742329_1505 (size=31229) 2024-12-04T14:58:41,382 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cbbafac124cb405994f73468217d2d80 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cbbafac124cb405994f73468217d2d80 2024-12-04T14:58:41,387 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/A of e727f8587f46eb6f1b53c466f4689322 into cbbafac124cb405994f73468217d2d80(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:41,387 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:41,387 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/A, priority=12, startTime=1733324321176; duration=0sec 2024-12-04T14:58:41,387 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:41,387 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:A 2024-12-04T14:58:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324381392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:41,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742330_1506 (size=31105) 2024-12-04T14:58:41,422 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cfa5c427ddc14b50b3ffba9e488db742 2024-12-04T14:58:41,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742331_1507 (size=12275) 2024-12-04T14:58:41,460 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/7027cc679b8d4d9b882e8faa436d571a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/7027cc679b8d4d9b882e8faa436d571a 2024-12-04T14:58:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/bd4c0698f35045e8bc38ef7743cb999c is 50, key is test_row_0/B:col10/1733324320714/Put/seqid=0 2024-12-04T14:58:41,466 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/C of e727f8587f46eb6f1b53c466f4689322 into 7027cc679b8d4d9b882e8faa436d571a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:41,466 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:41,466 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/C, priority=12, startTime=1733324321176; duration=0sec 2024-12-04T14:58:41,466 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:41,466 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:C 2024-12-04T14:58:41,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742332_1508 (size=12151) 2024-12-04T14:58:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324381505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T14:58:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324381715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:41,874 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/bd4c0698f35045e8bc38ef7743cb999c 2024-12-04T14:58:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/11ea3e9084c14543b7af448518045224 is 50, key is test_row_0/C:col10/1733324320714/Put/seqid=0 2024-12-04T14:58:41,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742333_1509 (size=12151) 2024-12-04T14:58:41,894 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/11ea3e9084c14543b7af448518045224 2024-12-04T14:58:41,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cfa5c427ddc14b50b3ffba9e488db742 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cfa5c427ddc14b50b3ffba9e488db742 2024-12-04T14:58:41,911 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cfa5c427ddc14b50b3ffba9e488db742, entries=150, sequenceid=153, filesize=30.4 K 2024-12-04T14:58:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/bd4c0698f35045e8bc38ef7743cb999c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/bd4c0698f35045e8bc38ef7743cb999c 2024-12-04T14:58:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,925 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/bd4c0698f35045e8bc38ef7743cb999c, entries=150, sequenceid=153, filesize=11.9 K 2024-12-04T14:58:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/11ea3e9084c14543b7af448518045224 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/11ea3e9084c14543b7af448518045224 2024-12-04T14:58:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,934 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/11ea3e9084c14543b7af448518045224, entries=150, sequenceid=153, filesize=11.9 K 2024-12-04T14:58:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,935 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e727f8587f46eb6f1b53c466f4689322 in 741ms, sequenceid=153, compaction requested=false 2024-12-04T14:58:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-04T14:58:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-04T14:58:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-04T14:58:41,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 897 msec 2024-12-04T14:58:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 903 msec 2024-12-04T14:58:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:42,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-04T14:58:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d8bd1b85095749448fda15eca075837d_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324321377/Put/seqid=0 2024-12-04T14:58:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742334_1510 (size=12304) 2024-12-04T14:58:42,107 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:42,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d8bd1b85095749448fda15eca075837d_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d8bd1b85095749448fda15eca075837d_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:42,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cc2994fa05c841b2b73d56643555380e, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:42,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cc2994fa05c841b2b73d56643555380e is 175, key is test_row_0/A:col10/1733324321377/Put/seqid=0 2024-12-04T14:58:42,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742335_1511 (size=31105) 2024-12-04T14:58:42,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324382145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T14:58:42,176 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:42,176 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-04T14:58:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T14:58:42,178 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:42,179 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:42,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:42,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324382255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:42,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T14:58:42,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T14:58:42,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:42,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:42,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:42,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324382475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:42,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T14:58:42,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:42,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:42,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:42,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T14:58:42,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cc2994fa05c841b2b73d56643555380e 2024-12-04T14:58:42,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/8d2c350a33c2415fb1d788390fd9d3c8 is 50, key is test_row_0/B:col10/1733324321377/Put/seqid=0 2024-12-04T14:58:42,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742336_1512 (size=12151) 2024-12-04T14:58:42,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T14:58:42,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:42,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:42,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:42,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324382785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:42,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T14:58:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:42,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T14:58:42,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T14:58:42,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:42,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:42,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:42,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:43,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/8d2c350a33c2415fb1d788390fd9d3c8 2024-12-04T14:58:43,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/4e97adc49b5942018950f888ca13608d is 50, key is test_row_0/C:col10/1733324321377/Put/seqid=0 2024-12-04T14:58:43,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742337_1513 (size=12151) 2024-12-04T14:58:43,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/4e97adc49b5942018950f888ca13608d 2024-12-04T14:58:43,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/cc2994fa05c841b2b73d56643555380e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cc2994fa05c841b2b73d56643555380e 2024-12-04T14:58:43,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cc2994fa05c841b2b73d56643555380e, entries=150, sequenceid=168, filesize=30.4 K 2024-12-04T14:58:43,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/8d2c350a33c2415fb1d788390fd9d3c8 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8d2c350a33c2415fb1d788390fd9d3c8 2024-12-04T14:58:43,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8d2c350a33c2415fb1d788390fd9d3c8, entries=150, sequenceid=168, filesize=11.9 K 2024-12-04T14:58:43,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/4e97adc49b5942018950f888ca13608d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/4e97adc49b5942018950f888ca13608d 2024-12-04T14:58:43,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/4e97adc49b5942018950f888ca13608d, entries=150, sequenceid=168, filesize=11.9 K 2024-12-04T14:58:43,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e727f8587f46eb6f1b53c466f4689322 in 1042ms, sequenceid=168, compaction requested=true 2024-12-04T14:58:43,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:43,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:43,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:43,072 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:43,072 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:43,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:43,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:43,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:43,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:43,074 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:43,074 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:43,074 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/B is initiating minor compaction (all files) 2024-12-04T14:58:43,074 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/A is initiating minor compaction (all files) 2024-12-04T14:58:43,074 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/A in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:43,074 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/B in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:43,074 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cbbafac124cb405994f73468217d2d80, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cfa5c427ddc14b50b3ffba9e488db742, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cc2994fa05c841b2b73d56643555380e] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=91.2 K 2024-12-04T14:58:43,074 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8261d2db54614a9baa7a527db8e1251f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/bd4c0698f35045e8bc38ef7743cb999c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8d2c350a33c2415fb1d788390fd9d3c8] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=35.7 K 2024-12-04T14:58:43,074 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:43,074 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cbbafac124cb405994f73468217d2d80, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cfa5c427ddc14b50b3ffba9e488db742, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cc2994fa05c841b2b73d56643555380e] 2024-12-04T14:58:43,074 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8261d2db54614a9baa7a527db8e1251f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733324319421 2024-12-04T14:58:43,074 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting bd4c0698f35045e8bc38ef7743cb999c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733324320700 2024-12-04T14:58:43,075 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbbafac124cb405994f73468217d2d80, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733324319421 2024-12-04T14:58:43,075 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2c350a33c2415fb1d788390fd9d3c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733324321377 2024-12-04T14:58:43,076 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfa5c427ddc14b50b3ffba9e488db742, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733324320700 2024-12-04T14:58:43,078 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc2994fa05c841b2b73d56643555380e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733324321377 2024-12-04T14:58:43,088 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:43,088 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/78279921a05548079a73922f2724c093 is 50, key is test_row_0/B:col10/1733324321377/Put/seqid=0 2024-12-04T14:58:43,100 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:43,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T14:58:43,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:43,102 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-04T14:58:43,104 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120410f25e7adad2441b8e2e563d77bcdab9_e727f8587f46eb6f1b53c466f4689322 store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:43,106 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120410f25e7adad2441b8e2e563d77bcdab9_e727f8587f46eb6f1b53c466f4689322, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:43,106 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120410f25e7adad2441b8e2e563d77bcdab9_e727f8587f46eb6f1b53c466f4689322 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:43,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742338_1514 (size=12527) 2024-12-04T14:58:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048481e9ec75554a79a4e73110834ba92a_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324322137/Put/seqid=0 2024-12-04T14:58:43,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742339_1515 (size=4469) 2024-12-04T14:58:43,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742340_1516 (size=12304) 2024-12-04T14:58:43,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T14:58:43,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324383341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:43,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:43,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324383455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:43,552 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#A#compaction#436 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:43,553 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/25113c5e8f434282a31da11823e6eec8 is 175, key is test_row_0/A:col10/1733324321377/Put/seqid=0 2024-12-04T14:58:43,570 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/78279921a05548079a73922f2724c093 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/78279921a05548079a73922f2724c093 2024-12-04T14:58:43,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742341_1517 (size=31481) 2024-12-04T14:58:43,578 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/B of e727f8587f46eb6f1b53c466f4689322 into 78279921a05548079a73922f2724c093(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:43,578 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:43,578 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/B, priority=13, startTime=1733324323072; duration=0sec 2024-12-04T14:58:43,578 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:43,578 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:B 2024-12-04T14:58:43,578 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:43,579 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:43,579 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/C is initiating minor compaction (all files) 2024-12-04T14:58:43,579 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/C in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:43,579 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/7027cc679b8d4d9b882e8faa436d571a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/11ea3e9084c14543b7af448518045224, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/4e97adc49b5942018950f888ca13608d] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=35.7 K 2024-12-04T14:58:43,580 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7027cc679b8d4d9b882e8faa436d571a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733324319421 2024-12-04T14:58:43,580 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 11ea3e9084c14543b7af448518045224, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733324320700 2024-12-04T14:58:43,581 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e97adc49b5942018950f888ca13608d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733324321377 2024-12-04T14:58:43,582 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/25113c5e8f434282a31da11823e6eec8 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/25113c5e8f434282a31da11823e6eec8 2024-12-04T14:58:43,587 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/A of e727f8587f46eb6f1b53c466f4689322 into 25113c5e8f434282a31da11823e6eec8(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:43,587 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:43,587 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/A, priority=13, startTime=1733324323072; duration=0sec 2024-12-04T14:58:43,587 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:43,587 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:A 2024-12-04T14:58:43,589 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#C#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:43,590 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/7521b52d9106411aa4f64ea857e2ca1e is 50, key is test_row_0/C:col10/1733324321377/Put/seqid=0 2024-12-04T14:58:43,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742342_1518 (size=12527) 2024-12-04T14:58:43,619 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/7521b52d9106411aa4f64ea857e2ca1e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/7521b52d9106411aa4f64ea857e2ca1e 2024-12-04T14:58:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:43,630 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/C of e727f8587f46eb6f1b53c466f4689322 into 7521b52d9106411aa4f64ea857e2ca1e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:43,630 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:43,630 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/C, priority=13, startTime=1733324323073; duration=0sec 2024-12-04T14:58:43,630 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:43,630 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:C 2024-12-04T14:58:43,634 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048481e9ec75554a79a4e73110834ba92a_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048481e9ec75554a79a4e73110834ba92a_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:43,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/52f27b6d8d1f45a3b3aaa1c946a076e2, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/52f27b6d8d1f45a3b3aaa1c946a076e2 is 175, key is test_row_0/A:col10/1733324322137/Put/seqid=0 2024-12-04T14:58:43,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324383665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:43,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742343_1519 (size=31105) 2024-12-04T14:58:43,677 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/52f27b6d8d1f45a3b3aaa1c946a076e2 2024-12-04T14:58:43,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/153f9a97c3da421dbd59f8074178919d is 50, key is test_row_0/B:col10/1733324322137/Put/seqid=0 2024-12-04T14:58:43,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742344_1520 (size=12151) 2024-12-04T14:58:43,715 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/153f9a97c3da421dbd59f8074178919d 2024-12-04T14:58:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/06c593e4894743ec8d04f15c2bfd59c3 is 50, key is test_row_0/C:col10/1733324322137/Put/seqid=0 2024-12-04T14:58:43,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742345_1521 (size=12151) 2024-12-04T14:58:43,760 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/06c593e4894743ec8d04f15c2bfd59c3 2024-12-04T14:58:43,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/52f27b6d8d1f45a3b3aaa1c946a076e2 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/52f27b6d8d1f45a3b3aaa1c946a076e2 2024-12-04T14:58:43,773 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/52f27b6d8d1f45a3b3aaa1c946a076e2, entries=150, sequenceid=192, filesize=30.4 K 2024-12-04T14:58:43,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/153f9a97c3da421dbd59f8074178919d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/153f9a97c3da421dbd59f8074178919d 2024-12-04T14:58:43,777 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/153f9a97c3da421dbd59f8074178919d, entries=150, sequenceid=192, filesize=11.9 K 2024-12-04T14:58:43,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/06c593e4894743ec8d04f15c2bfd59c3 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/06c593e4894743ec8d04f15c2bfd59c3 2024-12-04T14:58:43,782 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/06c593e4894743ec8d04f15c2bfd59c3, entries=150, sequenceid=192, filesize=11.9 K 2024-12-04T14:58:43,783 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e727f8587f46eb6f1b53c466f4689322 in 681ms, sequenceid=192, compaction requested=false 2024-12-04T14:58:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-04T14:58:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-04T14:58:43,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-04T14:58:43,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6060 sec 2024-12-04T14:58:43,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.6130 sec 2024-12-04T14:58:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:43,978 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:43,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-04T14:58:44,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204fb6a6157fe3a4b4ea8eda76033fe0baa_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324323313/Put/seqid=0 2024-12-04T14:58:44,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742346_1522 (size=12304) 2024-12-04T14:58:44,044 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:44,054 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204fb6a6157fe3a4b4ea8eda76033fe0baa_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204fb6a6157fe3a4b4ea8eda76033fe0baa_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:44,055 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0f93e09c82564d2e93a4adffad21af6e, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:44,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0f93e09c82564d2e93a4adffad21af6e is 175, key is test_row_0/A:col10/1733324323313/Put/seqid=0 2024-12-04T14:58:44,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742347_1523 (size=31105) 2024-12-04T14:58:44,114 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0f93e09c82564d2e93a4adffad21af6e 2024-12-04T14:58:44,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/d63555ee20244b25a7fd4fe408dbf8ea is 50, key is test_row_0/B:col10/1733324323313/Put/seqid=0 2024-12-04T14:58:44,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324384133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:44,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742348_1524 (size=12151) 2024-12-04T14:58:44,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324384245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T14:58:44,336 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:44,336 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-04T14:58:44,340 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T14:58:44,341 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:44,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T14:58:44,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324384455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:44,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T14:58:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/d63555ee20244b25a7fd4fe408dbf8ea 2024-12-04T14:58:44,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/0173bf2a88da475da71559747b98c7f7 is 50, key is test_row_0/C:col10/1733324323313/Put/seqid=0 2024-12-04T14:58:44,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T14:58:44,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:44,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742349_1525 (size=12151) 2024-12-04T14:58:44,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/0173bf2a88da475da71559747b98c7f7 2024-12-04T14:58:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T14:58:44,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0f93e09c82564d2e93a4adffad21af6e as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0f93e09c82564d2e93a4adffad21af6e 2024-12-04T14:58:44,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0f93e09c82564d2e93a4adffad21af6e, entries=150, sequenceid=208, filesize=30.4 K 2024-12-04T14:58:44,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/d63555ee20244b25a7fd4fe408dbf8ea as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/d63555ee20244b25a7fd4fe408dbf8ea 2024-12-04T14:58:44,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/d63555ee20244b25a7fd4fe408dbf8ea, entries=150, sequenceid=208, filesize=11.9 K 2024-12-04T14:58:44,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/0173bf2a88da475da71559747b98c7f7 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/0173bf2a88da475da71559747b98c7f7 2024-12-04T14:58:44,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/0173bf2a88da475da71559747b98c7f7, entries=150, sequenceid=208, filesize=11.9 K 2024-12-04T14:58:44,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e727f8587f46eb6f1b53c466f4689322 in 712ms, sequenceid=208, compaction requested=true 2024-12-04T14:58:44,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:44,690 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:44,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:44,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:44,691 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:44,691 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:44,691 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/A is initiating minor compaction (all files) 2024-12-04T14:58:44,691 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/A in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,691 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/25113c5e8f434282a31da11823e6eec8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/52f27b6d8d1f45a3b3aaa1c946a076e2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0f93e09c82564d2e93a4adffad21af6e] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=91.5 K 2024-12-04T14:58:44,691 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,691 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/25113c5e8f434282a31da11823e6eec8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/52f27b6d8d1f45a3b3aaa1c946a076e2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0f93e09c82564d2e93a4adffad21af6e] 2024-12-04T14:58:44,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:44,692 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25113c5e8f434282a31da11823e6eec8, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733324321377 2024-12-04T14:58:44,692 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52f27b6d8d1f45a3b3aaa1c946a076e2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733324322120 2024-12-04T14:58:44,693 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:44,693 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/B is initiating minor compaction (all files) 2024-12-04T14:58:44,693 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/B in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,693 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/78279921a05548079a73922f2724c093, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/153f9a97c3da421dbd59f8074178919d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/d63555ee20244b25a7fd4fe408dbf8ea] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=36.0 K 2024-12-04T14:58:44,693 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f93e09c82564d2e93a4adffad21af6e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733324323313 2024-12-04T14:58:44,693 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 78279921a05548079a73922f2724c093, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733324321377 2024-12-04T14:58:44,694 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 153f9a97c3da421dbd59f8074178919d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733324322120 2024-12-04T14:58:44,694 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting d63555ee20244b25a7fd4fe408dbf8ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733324323313 2024-12-04T14:58:44,711 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:44,711 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/78699dec8aef45288a2ad7251d88bd12 is 50, key is test_row_0/B:col10/1733324323313/Put/seqid=0 2024-12-04T14:58:44,717 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:44,742 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412044ad09456c36540e2b8dabbaa14ef05ff_e727f8587f46eb6f1b53c466f4689322 store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:44,743 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412044ad09456c36540e2b8dabbaa14ef05ff_e727f8587f46eb6f1b53c466f4689322, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:44,744 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044ad09456c36540e2b8dabbaa14ef05ff_e727f8587f46eb6f1b53c466f4689322 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:44,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742350_1526 (size=12629) 2024-12-04T14:58:44,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:44,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-04T14:58:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:44,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T14:58:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204be90b16237034bf5bea02341ff5c81bf_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324324131/Put/seqid=0 2024-12-04T14:58:44,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742351_1527 (size=4469) 2024-12-04T14:58:44,827 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#A#compaction#445 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:44,828 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/7c4d964f9dd94b18898ba6e71eb18346 is 175, key is test_row_0/A:col10/1733324323313/Put/seqid=0 2024-12-04T14:58:44,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324384849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:44,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742352_1528 (size=14794) 2024-12-04T14:58:44,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742353_1529 (size=31583) 2024-12-04T14:58:44,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T14:58:44,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:44,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:44,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T14:58:44,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324384965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:45,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T14:58:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:45,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324385174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:45,180 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/78699dec8aef45288a2ad7251d88bd12 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/78699dec8aef45288a2ad7251d88bd12 2024-12-04T14:58:45,186 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/B of e727f8587f46eb6f1b53c466f4689322 into 78699dec8aef45288a2ad7251d88bd12(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:45,186 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:45,186 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/B, priority=13, startTime=1733324324691; duration=0sec 2024-12-04T14:58:45,186 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:45,186 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:B 2024-12-04T14:58:45,186 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:45,188 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:45,188 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/C is initiating minor compaction (all files) 2024-12-04T14:58:45,188 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/C in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:45,189 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/7521b52d9106411aa4f64ea857e2ca1e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/06c593e4894743ec8d04f15c2bfd59c3, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/0173bf2a88da475da71559747b98c7f7] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=36.0 K 2024-12-04T14:58:45,189 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 7521b52d9106411aa4f64ea857e2ca1e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733324321377 2024-12-04T14:58:45,190 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 06c593e4894743ec8d04f15c2bfd59c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733324322120 2024-12-04T14:58:45,190 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 0173bf2a88da475da71559747b98c7f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733324323313 2024-12-04T14:58:45,198 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#C#compaction#447 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:45,199 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/b8490b2d2bc84bb6a2ae59b2884b3d5d is 50, key is test_row_0/C:col10/1733324323313/Put/seqid=0 2024-12-04T14:58:45,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742354_1530 (size=12629) 2024-12-04T14:58:45,248 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/b8490b2d2bc84bb6a2ae59b2884b3d5d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b8490b2d2bc84bb6a2ae59b2884b3d5d 2024-12-04T14:58:45,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T14:58:45,263 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/C of e727f8587f46eb6f1b53c466f4689322 into b8490b2d2bc84bb6a2ae59b2884b3d5d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:45,263 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:45,263 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/C, priority=13, startTime=1733324324692; duration=0sec 2024-12-04T14:58:45,263 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:45,263 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:C 2024-12-04T14:58:45,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:45,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:45,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:45,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,267 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,272 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204be90b16237034bf5bea02341ff5c81bf_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204be90b16237034bf5bea02341ff5c81bf_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:45,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/a40357a421c647c8b6cc6090285e8dfe, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:45,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/a40357a421c647c8b6cc6090285e8dfe is 175, key is test_row_0/A:col10/1733324324131/Put/seqid=0 2024-12-04T14:58:45,284 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/7c4d964f9dd94b18898ba6e71eb18346 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7c4d964f9dd94b18898ba6e71eb18346 2024-12-04T14:58:45,289 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/A of e727f8587f46eb6f1b53c466f4689322 into 7c4d964f9dd94b18898ba6e71eb18346(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:45,289 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:45,289 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/A, priority=13, startTime=1733324324690; duration=0sec 2024-12-04T14:58:45,289 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:45,289 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:A 2024-12-04T14:58:45,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742355_1531 (size=39749) 2024-12-04T14:58:45,316 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/a40357a421c647c8b6cc6090285e8dfe 2024-12-04T14:58:45,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b13d965a7bdc4545a98ac2b8f4f9f2fa is 50, key is test_row_0/B:col10/1733324324131/Put/seqid=0 2024-12-04T14:58:45,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742356_1532 (size=12151) 2024-12-04T14:58:45,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b13d965a7bdc4545a98ac2b8f4f9f2fa 2024-12-04T14:58:45,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T14:58:45,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:45,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:45,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/28ce659403a64434af7efb432dd4f4fc is 50, key is test_row_0/C:col10/1733324324131/Put/seqid=0 2024-12-04T14:58:45,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742357_1533 (size=12151) 2024-12-04T14:58:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T14:58:45,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/28ce659403a64434af7efb432dd4f4fc 2024-12-04T14:58:45,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/a40357a421c647c8b6cc6090285e8dfe as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/a40357a421c647c8b6cc6090285e8dfe 2024-12-04T14:58:45,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324385485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:45,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/a40357a421c647c8b6cc6090285e8dfe, entries=200, sequenceid=232, filesize=38.8 K 2024-12-04T14:58:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b13d965a7bdc4545a98ac2b8f4f9f2fa as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b13d965a7bdc4545a98ac2b8f4f9f2fa 2024-12-04T14:58:45,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b13d965a7bdc4545a98ac2b8f4f9f2fa, entries=150, sequenceid=232, filesize=11.9 K 2024-12-04T14:58:45,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/28ce659403a64434af7efb432dd4f4fc as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/28ce659403a64434af7efb432dd4f4fc 2024-12-04T14:58:45,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/28ce659403a64434af7efb432dd4f4fc, entries=150, sequenceid=232, filesize=11.9 K 2024-12-04T14:58:45,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e727f8587f46eb6f1b53c466f4689322 in 727ms, sequenceid=232, compaction requested=false 2024-12-04T14:58:45,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:45,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T14:58:45,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:45,572 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-04T14:58:45,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048662466828c74cd7af5bcdcb68f5d27c_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324324799/Put/seqid=0 2024-12-04T14:58:45,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742358_1534 (size=12304) 2024-12-04T14:58:45,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:45,617 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048662466828c74cd7af5bcdcb68f5d27c_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048662466828c74cd7af5bcdcb68f5d27c_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:45,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/6e2ca07774f2449ba3c653bc23c52595, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:45,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/6e2ca07774f2449ba3c653bc23c52595 is 175, key is test_row_0/A:col10/1733324324799/Put/seqid=0 2024-12-04T14:58:45,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742359_1535 (size=31105) 2024-12-04T14:58:45,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:46,029 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/6e2ca07774f2449ba3c653bc23c52595 2024-12-04T14:58:46,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/e58c779e688a42b6b08424d5d5306514 is 50, key is test_row_0/B:col10/1733324324799/Put/seqid=0 2024-12-04T14:58:46,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742360_1536 (size=12151) 2024-12-04T14:58:46,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324386125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:46,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324386234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:46,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324386445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T14:58:46,490 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/e58c779e688a42b6b08424d5d5306514 2024-12-04T14:58:46,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/2e823d35633e4073ba16be3b3c0b9ee5 is 50, key is test_row_0/C:col10/1733324324799/Put/seqid=0 2024-12-04T14:58:46,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742361_1537 (size=12151) 2024-12-04T14:58:46,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324386756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:46,935 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/2e823d35633e4073ba16be3b3c0b9ee5 2024-12-04T14:58:46,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/6e2ca07774f2449ba3c653bc23c52595 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/6e2ca07774f2449ba3c653bc23c52595 2024-12-04T14:58:46,944 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/6e2ca07774f2449ba3c653bc23c52595, entries=150, sequenceid=247, filesize=30.4 K 2024-12-04T14:58:46,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/e58c779e688a42b6b08424d5d5306514 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/e58c779e688a42b6b08424d5d5306514 2024-12-04T14:58:46,948 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/e58c779e688a42b6b08424d5d5306514, entries=150, sequenceid=247, filesize=11.9 K 2024-12-04T14:58:46,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/2e823d35633e4073ba16be3b3c0b9ee5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2e823d35633e4073ba16be3b3c0b9ee5 2024-12-04T14:58:46,954 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2e823d35633e4073ba16be3b3c0b9ee5, entries=150, sequenceid=247, filesize=11.9 K 2024-12-04T14:58:46,955 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e727f8587f46eb6f1b53c466f4689322 in 1383ms, sequenceid=247, compaction requested=true 2024-12-04T14:58:46,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:46,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:46,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-04T14:58:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-04T14:58:46,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-04T14:58:46,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6170 sec 2024-12-04T14:58:46,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.6260 sec 2024-12-04T14:58:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:47,268 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:47,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:58:47,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048a121e7f42e440ce95a691f773c928ce_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324326123/Put/seqid=0 2024-12-04T14:58:47,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324387313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:47,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742362_1538 (size=12454) 2024-12-04T14:58:47,334 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:47,340 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412048a121e7f42e440ce95a691f773c928ce_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048a121e7f42e440ce95a691f773c928ce_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:47,343 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/fd805af59c9643dca471865ebd443a59, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:47,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/fd805af59c9643dca471865ebd443a59 is 175, key is test_row_0/A:col10/1733324326123/Put/seqid=0 2024-12-04T14:58:47,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742363_1539 (size=31255) 2024-12-04T14:58:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324387425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:47,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324387635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:47,777 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/fd805af59c9643dca471865ebd443a59 2024-12-04T14:58:47,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/85383e27fdac4209aa40d4cba477c652 is 50, key is test_row_0/B:col10/1733324326123/Put/seqid=0 2024-12-04T14:58:47,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742364_1540 (size=12301) 2024-12-04T14:58:47,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/85383e27fdac4209aa40d4cba477c652 2024-12-04T14:58:47,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/c300c249b7424f4fb501938466b76026 is 50, key is test_row_0/C:col10/1733324326123/Put/seqid=0 2024-12-04T14:58:47,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742365_1541 (size=12301) 2024-12-04T14:58:47,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/c300c249b7424f4fb501938466b76026 2024-12-04T14:58:47,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/fd805af59c9643dca471865ebd443a59 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/fd805af59c9643dca471865ebd443a59 2024-12-04T14:58:47,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/fd805af59c9643dca471865ebd443a59, entries=150, sequenceid=272, filesize=30.5 K 2024-12-04T14:58:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/85383e27fdac4209aa40d4cba477c652 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/85383e27fdac4209aa40d4cba477c652 2024-12-04T14:58:47,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/85383e27fdac4209aa40d4cba477c652, entries=150, sequenceid=272, filesize=12.0 K 2024-12-04T14:58:47,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/c300c249b7424f4fb501938466b76026 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/c300c249b7424f4fb501938466b76026 2024-12-04T14:58:47,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/c300c249b7424f4fb501938466b76026, entries=150, sequenceid=272, filesize=12.0 K 2024-12-04T14:58:47,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e727f8587f46eb6f1b53c466f4689322 in 634ms, sequenceid=272, compaction requested=true 2024-12-04T14:58:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:47,903 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:47,903 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:47,904 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:47,904 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133692 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:47,904 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/A is initiating minor compaction (all files) 2024-12-04T14:58:47,904 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/B is initiating minor compaction (all files) 2024-12-04T14:58:47,905 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/B in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:47,905 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/A in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:47,905 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/78699dec8aef45288a2ad7251d88bd12, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b13d965a7bdc4545a98ac2b8f4f9f2fa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/e58c779e688a42b6b08424d5d5306514, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/85383e27fdac4209aa40d4cba477c652] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=48.1 K 2024-12-04T14:58:47,905 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7c4d964f9dd94b18898ba6e71eb18346, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/a40357a421c647c8b6cc6090285e8dfe, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/6e2ca07774f2449ba3c653bc23c52595, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/fd805af59c9643dca471865ebd443a59] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=130.6 K 2024-12-04T14:58:47,905 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:47,905 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7c4d964f9dd94b18898ba6e71eb18346, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/a40357a421c647c8b6cc6090285e8dfe, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/6e2ca07774f2449ba3c653bc23c52595, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/fd805af59c9643dca471865ebd443a59] 2024-12-04T14:58:47,905 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 78699dec8aef45288a2ad7251d88bd12, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733324323313 2024-12-04T14:58:47,905 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c4d964f9dd94b18898ba6e71eb18346, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733324323313 2024-12-04T14:58:47,905 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b13d965a7bdc4545a98ac2b8f4f9f2fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733324324102 2024-12-04T14:58:47,908 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting a40357a421c647c8b6cc6090285e8dfe, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733324324088 2024-12-04T14:58:47,908 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting e58c779e688a42b6b08424d5d5306514, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733324324799 2024-12-04T14:58:47,909 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e2ca07774f2449ba3c653bc23c52595, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733324324799 2024-12-04T14:58:47,909 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 85383e27fdac4209aa40d4cba477c652, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733324326112 2024-12-04T14:58:47,909 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd805af59c9643dca471865ebd443a59, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733324326112 2024-12-04T14:58:47,944 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#B#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:47,944 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/6df48f929cad45a1a2485cb8d669ba22 is 50, key is test_row_0/B:col10/1733324326123/Put/seqid=0 2024-12-04T14:58:47,946 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:47,948 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:47,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-04T14:58:47,953 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204e12b0bc220644c93a8bce692598305bd_e727f8587f46eb6f1b53c466f4689322 store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:47,963 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204e12b0bc220644c93a8bce692598305bd_e727f8587f46eb6f1b53c466f4689322, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:47,964 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e12b0bc220644c93a8bce692598305bd_e727f8587f46eb6f1b53c466f4689322 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:48,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f72480d7158a4d6a82aefbc8b89ab9f2_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324327278/Put/seqid=0 2024-12-04T14:58:48,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742367_1543 (size=4469) 2024-12-04T14:58:48,025 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#A#compaction#457 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:48,026 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/8db77511d23749598a6bdd4fcdf97e53 is 175, key is test_row_0/A:col10/1733324326123/Put/seqid=0 2024-12-04T14:58:48,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742366_1542 (size=12915) 2024-12-04T14:58:48,046 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/6df48f929cad45a1a2485cb8d669ba22 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/6df48f929cad45a1a2485cb8d669ba22 2024-12-04T14:58:48,055 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/B of e727f8587f46eb6f1b53c466f4689322 into 6df48f929cad45a1a2485cb8d669ba22(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:48,055 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:48,055 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/B, priority=12, startTime=1733324327903; duration=0sec 2024-12-04T14:58:48,056 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:48,056 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:B 2024-12-04T14:58:48,056 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T14:58:48,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742368_1544 (size=12454) 2024-12-04T14:58:48,059 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:48,060 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T14:58:48,060 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/C is initiating minor compaction (all files) 2024-12-04T14:58:48,060 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/C in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:48,060 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b8490b2d2bc84bb6a2ae59b2884b3d5d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/28ce659403a64434af7efb432dd4f4fc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2e823d35633e4073ba16be3b3c0b9ee5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/c300c249b7424f4fb501938466b76026] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=48.1 K 2024-12-04T14:58:48,063 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting b8490b2d2bc84bb6a2ae59b2884b3d5d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733324323313 2024-12-04T14:58:48,064 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ce659403a64434af7efb432dd4f4fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733324324102 2024-12-04T14:58:48,066 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f72480d7158a4d6a82aefbc8b89ab9f2_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f72480d7158a4d6a82aefbc8b89ab9f2_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:48,066 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e823d35633e4073ba16be3b3c0b9ee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733324324799 2024-12-04T14:58:48,067 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting c300c249b7424f4fb501938466b76026, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733324326112 2024-12-04T14:58:48,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/51642eccdb5c4dacb7da83e8ff8f169d, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:48,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/51642eccdb5c4dacb7da83e8ff8f169d is 175, key is test_row_0/A:col10/1733324327278/Put/seqid=0 2024-12-04T14:58:48,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742369_1545 (size=31869) 2024-12-04T14:58:48,081 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/8db77511d23749598a6bdd4fcdf97e53 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8db77511d23749598a6bdd4fcdf97e53 2024-12-04T14:58:48,087 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/A of e727f8587f46eb6f1b53c466f4689322 into 8db77511d23749598a6bdd4fcdf97e53(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:48,087 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:48,087 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/A, priority=12, startTime=1733324327903; duration=0sec 2024-12-04T14:58:48,087 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:48,087 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:A 2024-12-04T14:58:48,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324388086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:48,106 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:48,107 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/3e5b690ee4fd4767a1de956d11da685f is 50, key is test_row_0/C:col10/1733324326123/Put/seqid=0 2024-12-04T14:58:48,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742370_1546 (size=31255) 2024-12-04T14:58:48,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/51642eccdb5c4dacb7da83e8ff8f169d 2024-12-04T14:58:48,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/dc5207fadafc46ab80949aa98e8060eb is 50, key is test_row_0/B:col10/1733324327278/Put/seqid=0 2024-12-04T14:58:48,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742371_1547 (size=12915) 2024-12-04T14:58:48,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742372_1548 (size=12301) 2024-12-04T14:58:48,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/dc5207fadafc46ab80949aa98e8060eb 2024-12-04T14:58:48,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324388195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:48,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/bb3d1a6039f84261b587c8eedbb1dc0d is 50, key is test_row_0/C:col10/1733324327278/Put/seqid=0 2024-12-04T14:58:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:48,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742373_1549 (size=12301) 2024-12-04T14:58:48,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/bb3d1a6039f84261b587c8eedbb1dc0d 2024-12-04T14:58:48,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/51642eccdb5c4dacb7da83e8ff8f169d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/51642eccdb5c4dacb7da83e8ff8f169d 2024-12-04T14:58:48,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/51642eccdb5c4dacb7da83e8ff8f169d, entries=150, sequenceid=284, filesize=30.5 K 2024-12-04T14:58:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/dc5207fadafc46ab80949aa98e8060eb as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/dc5207fadafc46ab80949aa98e8060eb 2024-12-04T14:58:48,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/dc5207fadafc46ab80949aa98e8060eb, entries=150, sequenceid=284, filesize=12.0 K 2024-12-04T14:58:48,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/bb3d1a6039f84261b587c8eedbb1dc0d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/bb3d1a6039f84261b587c8eedbb1dc0d 2024-12-04T14:58:48,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/bb3d1a6039f84261b587c8eedbb1dc0d, entries=150, sequenceid=284, filesize=12.0 K 2024-12-04T14:58:48,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e727f8587f46eb6f1b53c466f4689322 in 328ms, sequenceid=284, compaction requested=false 2024-12-04T14:58:48,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:48,408 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:48,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:58:48,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120471a273af36464f44947db40e03766d7a_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324328407/Put/seqid=0 2024-12-04T14:58:48,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742374_1550 (size=12454) 2024-12-04T14:58:48,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324388472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T14:58:48,506 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:48,506 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-04T14:58:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T14:58:48,508 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:48,509 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:48,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:48,579 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/3e5b690ee4fd4767a1de956d11da685f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/3e5b690ee4fd4767a1de956d11da685f 2024-12-04T14:58:48,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324388585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:48,602 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/C of e727f8587f46eb6f1b53c466f4689322 into 3e5b690ee4fd4767a1de956d11da685f(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:48,602 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:48,602 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/C, priority=12, startTime=1733324327903; duration=0sec 2024-12-04T14:58:48,602 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:48,602 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:C 2024-12-04T14:58:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T14:58:48,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T14:58:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:48,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:48,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324388795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:48,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T14:58:48,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:48,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:48,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:48,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:48,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T14:58:48,844 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:48,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120471a273af36464f44947db40e03766d7a_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120471a273af36464f44947db40e03766d7a_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:48,848 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/31a9ab15520e468dbb40e57f7310337f, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:48,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/31a9ab15520e468dbb40e57f7310337f is 175, key is test_row_0/A:col10/1733324328407/Put/seqid=0 2024-12-04T14:58:48,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742375_1551 (size=31255) 2024-12-04T14:58:48,889 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/31a9ab15520e468dbb40e57f7310337f 2024-12-04T14:58:48,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/76fd0889a9ce458492f75964d439ec2f is 50, key is test_row_0/B:col10/1733324328407/Put/seqid=0 2024-12-04T14:58:48,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742376_1552 (size=12301) 2024-12-04T14:58:48,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T14:58:48,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:48,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:48,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:48,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:48,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324389105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:49,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T14:58:49,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:49,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T14:58:49,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T14:58:49,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:49,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/76fd0889a9ce458492f75964d439ec2f 2024-12-04T14:58:49,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/b79304bc3ff6415faf1eeda5ef4ccf30 is 50, key is test_row_0/C:col10/1733324328407/Put/seqid=0 2024-12-04T14:58:49,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742377_1553 (size=12301) 2024-12-04T14:58:49,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/b79304bc3ff6415faf1eeda5ef4ccf30 2024-12-04T14:58:49,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/31a9ab15520e468dbb40e57f7310337f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/31a9ab15520e468dbb40e57f7310337f 2024-12-04T14:58:49,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/31a9ab15520e468dbb40e57f7310337f, entries=150, sequenceid=311, filesize=30.5 K 2024-12-04T14:58:49,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/76fd0889a9ce458492f75964d439ec2f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/76fd0889a9ce458492f75964d439ec2f 2024-12-04T14:58:49,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/76fd0889a9ce458492f75964d439ec2f, entries=150, sequenceid=311, filesize=12.0 K 2024-12-04T14:58:49,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/b79304bc3ff6415faf1eeda5ef4ccf30 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b79304bc3ff6415faf1eeda5ef4ccf30 2024-12-04T14:58:49,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T14:58:49,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:49,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b79304bc3ff6415faf1eeda5ef4ccf30, entries=150, sequenceid=311, filesize=12.0 K 2024-12-04T14:58:49,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e727f8587f46eb6f1b53c466f4689322 in 1026ms, sequenceid=311, compaction requested=true 2024-12-04T14:58:49,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:49,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:49,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:49,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:49,434 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:49,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:49,434 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:49,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:49,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:49,435 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:49,435 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/B is initiating minor compaction (all files) 2024-12-04T14:58:49,435 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/B in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,435 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/6df48f929cad45a1a2485cb8d669ba22, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/dc5207fadafc46ab80949aa98e8060eb, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/76fd0889a9ce458492f75964d439ec2f] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=36.6 K 2024-12-04T14:58:49,435 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:49,435 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/A is initiating minor compaction (all files) 2024-12-04T14:58:49,436 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/A in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,436 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8db77511d23749598a6bdd4fcdf97e53, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/51642eccdb5c4dacb7da83e8ff8f169d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/31a9ab15520e468dbb40e57f7310337f] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=92.2 K 2024-12-04T14:58:49,436 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,436 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8db77511d23749598a6bdd4fcdf97e53, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/51642eccdb5c4dacb7da83e8ff8f169d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/31a9ab15520e468dbb40e57f7310337f] 2024-12-04T14:58:49,436 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df48f929cad45a1a2485cb8d669ba22, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733324326112 2024-12-04T14:58:49,436 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db77511d23749598a6bdd4fcdf97e53, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733324326112 2024-12-04T14:58:49,437 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc5207fadafc46ab80949aa98e8060eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733324327278 2024-12-04T14:58:49,437 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 51642eccdb5c4dacb7da83e8ff8f169d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733324327278 2024-12-04T14:58:49,437 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76fd0889a9ce458492f75964d439ec2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733324328076 2024-12-04T14:58:49,437 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 31a9ab15520e468dbb40e57f7310337f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733324328076 2024-12-04T14:58:49,453 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:49,453 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/c78ad3b1c4ce4a78a8a413ce032433c4 is 50, key is test_row_0/B:col10/1733324328407/Put/seqid=0 2024-12-04T14:58:49,465 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:49,481 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204ccd218fa1bc04993a55c71fd2adf05d2_e727f8587f46eb6f1b53c466f4689322 store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:49,483 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204ccd218fa1bc04993a55c71fd2adf05d2_e727f8587f46eb6f1b53c466f4689322, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:49,483 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ccd218fa1bc04993a55c71fd2adf05d2_e727f8587f46eb6f1b53c466f4689322 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:49,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742378_1554 (size=13017) 2024-12-04T14:58:49,507 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/c78ad3b1c4ce4a78a8a413ce032433c4 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/c78ad3b1c4ce4a78a8a413ce032433c4 2024-12-04T14:58:49,514 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/B of e727f8587f46eb6f1b53c466f4689322 into c78ad3b1c4ce4a78a8a413ce032433c4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:49,514 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:49,514 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/B, priority=13, startTime=1733324329434; duration=0sec 2024-12-04T14:58:49,514 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:49,514 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:B 2024-12-04T14:58:49,514 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:49,515 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:49,515 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/C is initiating minor compaction (all files) 2024-12-04T14:58:49,515 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/C in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,516 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/3e5b690ee4fd4767a1de956d11da685f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/bb3d1a6039f84261b587c8eedbb1dc0d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b79304bc3ff6415faf1eeda5ef4ccf30] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=36.6 K 2024-12-04T14:58:49,516 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e5b690ee4fd4767a1de956d11da685f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733324326112 2024-12-04T14:58:49,516 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb3d1a6039f84261b587c8eedbb1dc0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733324327278 2024-12-04T14:58:49,516 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting b79304bc3ff6415faf1eeda5ef4ccf30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733324328076 2024-12-04T14:58:49,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742379_1555 (size=4469) 2024-12-04T14:58:49,533 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#A#compaction#466 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:49,534 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/5a8bb5b413764e939237aa613fe16899 is 175, key is test_row_0/A:col10/1733324328407/Put/seqid=0 2024-12-04T14:58:49,542 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:49,543 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/5c69953ff4334f939960266a7ad43f44 is 50, key is test_row_0/C:col10/1733324328407/Put/seqid=0 2024-12-04T14:58:49,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T14:58:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:49,587 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:58:49,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742380_1556 (size=31971) 2024-12-04T14:58:49,596 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/5a8bb5b413764e939237aa613fe16899 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/5a8bb5b413764e939237aa613fe16899 2024-12-04T14:58:49,603 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/A of e727f8587f46eb6f1b53c466f4689322 into 5a8bb5b413764e939237aa613fe16899(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:49,603 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:49,603 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/A, priority=13, startTime=1733324329434; duration=0sec 2024-12-04T14:58:49,603 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:49,603 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:A 2024-12-04T14:58:49,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742381_1557 (size=13017) 2024-12-04T14:58:49,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120410f57654685b468c944487c70ae9a94c_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324328436/Put/seqid=0 2024-12-04T14:58:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:49,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T14:58:49,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742382_1558 (size=12454) 2024-12-04T14:58:49,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324389793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:49,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324389905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:49,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:49,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:50,015 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/5c69953ff4334f939960266a7ad43f44 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/5c69953ff4334f939960266a7ad43f44 2024-12-04T14:58:50,019 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/C of e727f8587f46eb6f1b53c466f4689322 into 5c69953ff4334f939960266a7ad43f44(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:50,020 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:50,020 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/C, priority=13, startTime=1733324329434; duration=0sec 2024-12-04T14:58:50,020 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:50,020 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:C 2024-12-04T14:58:50,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:50,066 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120410f57654685b468c944487c70ae9a94c_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120410f57654685b468c944487c70ae9a94c_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:50,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0e1c339f73cd4bd3b9313c846a56c1e1, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:50,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0e1c339f73cd4bd3b9313c846a56c1e1 is 175, key is test_row_0/A:col10/1733324328436/Put/seqid=0 2024-12-04T14:58:50,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742383_1559 (size=31255) 2024-12-04T14:58:50,111 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0e1c339f73cd4bd3b9313c846a56c1e1 2024-12-04T14:58:50,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324390115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:50,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/20fc97f5d4fc4ccf95885ae416b0cea4 is 50, key is test_row_0/B:col10/1733324328436/Put/seqid=0 2024-12-04T14:58:50,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742384_1560 (size=12301) 2024-12-04T14:58:50,161 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/20fc97f5d4fc4ccf95885ae416b0cea4 2024-12-04T14:58:50,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/b3484ccf4c844ad194a7b01b0da62f6a is 50, key is test_row_0/C:col10/1733324328436/Put/seqid=0 2024-12-04T14:58:50,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742385_1561 (size=12301) 2024-12-04T14:58:50,186 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/b3484ccf4c844ad194a7b01b0da62f6a 2024-12-04T14:58:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/0e1c339f73cd4bd3b9313c846a56c1e1 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0e1c339f73cd4bd3b9313c846a56c1e1 2024-12-04T14:58:50,198 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0e1c339f73cd4bd3b9313c846a56c1e1, entries=150, sequenceid=324, filesize=30.5 K 2024-12-04T14:58:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/20fc97f5d4fc4ccf95885ae416b0cea4 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/20fc97f5d4fc4ccf95885ae416b0cea4 2024-12-04T14:58:50,205 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/20fc97f5d4fc4ccf95885ae416b0cea4, entries=150, sequenceid=324, filesize=12.0 K 2024-12-04T14:58:50,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/b3484ccf4c844ad194a7b01b0da62f6a as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b3484ccf4c844ad194a7b01b0da62f6a 2024-12-04T14:58:50,209 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b3484ccf4c844ad194a7b01b0da62f6a, entries=150, sequenceid=324, filesize=12.0 K 2024-12-04T14:58:50,211 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e727f8587f46eb6f1b53c466f4689322 in 623ms, sequenceid=324, compaction requested=false 2024-12-04T14:58:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-04T14:58:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-04T14:58:50,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-04T14:58:50,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7030 sec 2024-12-04T14:58:50,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.7130 sec 2024-12-04T14:58:50,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:50,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-04T14:58:50,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d4a5f2ca2a16487db16b44f78c837a0b_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324329791/Put/seqid=0 2024-12-04T14:58:50,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742386_1562 (size=12454) 2024-12-04T14:58:50,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324390463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:50,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:50,479 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d4a5f2ca2a16487db16b44f78c837a0b_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d4a5f2ca2a16487db16b44f78c837a0b_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:50,485 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/8d2a460c156f4ec197c29668d6099d32, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:50,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/8d2a460c156f4ec197c29668d6099d32 is 175, key is test_row_0/A:col10/1733324329791/Put/seqid=0 2024-12-04T14:58:50,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742387_1563 (size=31255) 2024-12-04T14:58:50,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324390575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T14:58:50,655 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:50,656 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T14:58:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-04T14:58:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T14:58:50,658 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T14:58:50,659 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T14:58:50,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T14:58:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T14:58:50,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324390785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:50,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:50,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34628 deadline: 1733324390875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:50,877 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_0' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34654 deadline: 1733324390905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:50,907 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:50,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/8d2a460c156f4ec197c29668d6099d32 2024-12-04T14:58:50,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34636 deadline: 1733324390925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:50,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34664 deadline: 1733324390925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:50,929 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:50,929 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to d214edae7bd2,34477,1733324158096 for 'test_row_1' in e727f8587f46eb6f1b53c466f4689322 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T14:58:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:50,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/4091337f35b74dd583b9b50ca8ea40d5 is 50, key is test_row_0/B:col10/1733324329791/Put/seqid=0 2024-12-04T14:58:50,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:50,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T14:58:50,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742388_1564 (size=12301) 2024-12-04T14:58:50,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/4091337f35b74dd583b9b50ca8ea40d5 2024-12-04T14:58:51,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/89e9371a91aa4274a5094c98d447227c is 50, key is test_row_0/C:col10/1733324329791/Put/seqid=0 2024-12-04T14:58:51,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742389_1565 (size=12301) 2024-12-04T14:58:51,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/89e9371a91aa4274a5094c98d447227c 2024-12-04T14:58:51,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/8d2a460c156f4ec197c29668d6099d32 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8d2a460c156f4ec197c29668d6099d32 2024-12-04T14:58:51,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8d2a460c156f4ec197c29668d6099d32, entries=150, sequenceid=352, filesize=30.5 K 2024-12-04T14:58:51,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/4091337f35b74dd583b9b50ca8ea40d5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/4091337f35b74dd583b9b50ca8ea40d5 2024-12-04T14:58:51,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/4091337f35b74dd583b9b50ca8ea40d5, entries=150, sequenceid=352, filesize=12.0 K 2024-12-04T14:58:51,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/89e9371a91aa4274a5094c98d447227c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/89e9371a91aa4274a5094c98d447227c 2024-12-04T14:58:51,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/89e9371a91aa4274a5094c98d447227c, entries=150, sequenceid=352, filesize=12.0 K 2024-12-04T14:58:51,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e727f8587f46eb6f1b53c466f4689322 in 649ms, sequenceid=352, compaction requested=true 2024-12-04T14:58:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:51,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T14:58:51,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:51,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T14:58:51,078 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:51,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:51,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e727f8587f46eb6f1b53c466f4689322:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T14:58:51,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T14:58:51,078 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:51,084 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:51,084 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/A is initiating minor compaction (all files) 2024-12-04T14:58:51,084 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:51,084 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/B is initiating minor compaction (all files) 2024-12-04T14:58:51,084 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/A in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,084 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/B in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,084 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/5a8bb5b413764e939237aa613fe16899, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0e1c339f73cd4bd3b9313c846a56c1e1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8d2a460c156f4ec197c29668d6099d32] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=92.3 K 2024-12-04T14:58:51,084 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/c78ad3b1c4ce4a78a8a413ce032433c4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/20fc97f5d4fc4ccf95885ae416b0cea4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/4091337f35b74dd583b9b50ca8ea40d5] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=36.7 K 2024-12-04T14:58:51,084 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,084 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. files: [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/5a8bb5b413764e939237aa613fe16899, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0e1c339f73cd4bd3b9313c846a56c1e1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8d2a460c156f4ec197c29668d6099d32] 2024-12-04T14:58:51,084 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting c78ad3b1c4ce4a78a8a413ce032433c4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733324328076 2024-12-04T14:58:51,085 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a8bb5b413764e939237aa613fe16899, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733324328076 2024-12-04T14:58:51,085 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20fc97f5d4fc4ccf95885ae416b0cea4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733324328436 2024-12-04T14:58:51,085 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1c339f73cd4bd3b9313c846a56c1e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733324328436 2024-12-04T14:58:51,085 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4091337f35b74dd583b9b50ca8ea40d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733324329785 2024-12-04T14:58:51,085 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2a460c156f4ec197c29668d6099d32, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733324329785 2024-12-04T14:58:51,101 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-04T14:58:51,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-04T14:58:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:51,104 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:51,109 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:51,109 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/92ba2118d00343729551ea5139ba0546 is 50, key is test_row_0/B:col10/1733324329791/Put/seqid=0 2024-12-04T14:58:51,113 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204b70864d9f28d423bb199efe962eb2d47_e727f8587f46eb6f1b53c466f4689322 store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:51,114 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204b70864d9f28d423bb199efe962eb2d47_e727f8587f46eb6f1b53c466f4689322, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:51,115 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b70864d9f28d423bb199efe962eb2d47_e727f8587f46eb6f1b53c466f4689322 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:51,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041cf89d957f944d7682b87ec2134e2e21_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324330454/Put/seqid=0 2024-12-04T14:58:51,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742390_1566 (size=13119) 2024-12-04T14:58:51,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742391_1567 (size=4469) 2024-12-04T14:58:51,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742392_1568 (size=12454) 2024-12-04T14:58:51,145 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:51,148 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041cf89d957f944d7682b87ec2134e2e21_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041cf89d957f944d7682b87ec2134e2e21_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:51,149 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/f301014e07934730a11387c0b64661d6, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:51,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/f301014e07934730a11387c0b64661d6 is 175, key is test_row_0/A:col10/1733324330454/Put/seqid=0 2024-12-04T14:58:51,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742393_1569 (size=31255) 2024-12-04T14:58:51,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 350 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324391189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:51,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:51,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T14:58:51,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324391295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:51,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:51,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:51,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 352 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324391505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:51,532 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/92ba2118d00343729551ea5139ba0546 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/92ba2118d00343729551ea5139ba0546 2024-12-04T14:58:51,535 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/B of e727f8587f46eb6f1b53c466f4689322 into 92ba2118d00343729551ea5139ba0546(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:51,535 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:51,536 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/B, priority=13, startTime=1733324331078; duration=0sec 2024-12-04T14:58:51,536 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T14:58:51,536 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:B 2024-12-04T14:58:51,536 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T14:58:51,536 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T14:58:51,536 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1541): e727f8587f46eb6f1b53c466f4689322/C is initiating minor compaction (all files) 2024-12-04T14:58:51,537 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e727f8587f46eb6f1b53c466f4689322/C in TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,537 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/5c69953ff4334f939960266a7ad43f44, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b3484ccf4c844ad194a7b01b0da62f6a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/89e9371a91aa4274a5094c98d447227c] into tmpdir=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp, totalSize=36.7 K 2024-12-04T14:58:51,537 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c69953ff4334f939960266a7ad43f44, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733324328076 2024-12-04T14:58:51,538 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3484ccf4c844ad194a7b01b0da62f6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733324328436 2024-12-04T14:58:51,539 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e9371a91aa4274a5094c98d447227c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733324329785 2024-12-04T14:58:51,540 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:51,541 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/26a8b23c0aae4827b29fc91a8caaba38 is 175, key is test_row_0/A:col10/1733324329791/Put/seqid=0 2024-12-04T14:58:51,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742394_1570 (size=32073) 2024-12-04T14:58:51,546 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e727f8587f46eb6f1b53c466f4689322#C#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T14:58:51,546 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/385906e9edcc4423a4e754330a5090fe is 50, key is test_row_0/C:col10/1733324329791/Put/seqid=0 2024-12-04T14:58:51,548 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/26a8b23c0aae4827b29fc91a8caaba38 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/26a8b23c0aae4827b29fc91a8caaba38 2024-12-04T14:58:51,552 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/A of e727f8587f46eb6f1b53c466f4689322 into 26a8b23c0aae4827b29fc91a8caaba38(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:51,552 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:51,552 INFO [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/A, priority=13, startTime=1733324331077; duration=0sec 2024-12-04T14:58:51,552 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:51,552 DEBUG [RS:0;d214edae7bd2:34477-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:A 2024-12-04T14:58:51,553 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/f301014e07934730a11387c0b64661d6 2024-12-04T14:58:51,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742395_1571 (size=13119) 2024-12-04T14:58:51,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b92c08e294724e768d5a88b1c28a741d is 50, key is test_row_0/B:col10/1733324330454/Put/seqid=0 2024-12-04T14:58:51,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742396_1572 (size=12301) 2024-12-04T14:58:51,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:51,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:51,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T14:58:51,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324391815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:51,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:51,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:51,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:51,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:51,956 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/385906e9edcc4423a4e754330a5090fe as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/385906e9edcc4423a4e754330a5090fe 2024-12-04T14:58:51,960 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e727f8587f46eb6f1b53c466f4689322/C of e727f8587f46eb6f1b53c466f4689322 into 385906e9edcc4423a4e754330a5090fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T14:58:51,960 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:51,960 INFO [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., storeName=e727f8587f46eb6f1b53c466f4689322/C, priority=13, startTime=1733324331078; duration=0sec 2024-12-04T14:58:51,960 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T14:58:51,960 DEBUG [RS:0;d214edae7bd2:34477-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e727f8587f46eb6f1b53c466f4689322:C 2024-12-04T14:58:51,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b92c08e294724e768d5a88b1c28a741d 2024-12-04T14:58:51,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/21b2d193514a40fba209310da4c2450b is 50, key is test_row_0/C:col10/1733324330454/Put/seqid=0 2024-12-04T14:58:51,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742397_1573 (size=12301) 2024-12-04T14:58:52,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:52,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:52,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:52,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:52,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:52,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:52,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:52,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:52,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:52,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T14:58:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34477 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1733324392331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 2024-12-04T14:58:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 , the old value is region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T14:58:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., hostname=d214edae7bd2,34477,1733324158096, seqNum=5 because the exception is null or not the one we care about 2024-12-04T14:58:52,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:52,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:52,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:52,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:52,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:52,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T14:58:52,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/21b2d193514a40fba209310da4c2450b 2024-12-04T14:58:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/f301014e07934730a11387c0b64661d6 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/f301014e07934730a11387c0b64661d6 2024-12-04T14:58:52,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/f301014e07934730a11387c0b64661d6, entries=150, sequenceid=363, filesize=30.5 K 2024-12-04T14:58:52,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/b92c08e294724e768d5a88b1c28a741d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b92c08e294724e768d5a88b1c28a741d 2024-12-04T14:58:52,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b92c08e294724e768d5a88b1c28a741d, entries=150, sequenceid=363, filesize=12.0 K 2024-12-04T14:58:52,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/21b2d193514a40fba209310da4c2450b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/21b2d193514a40fba209310da4c2450b 2024-12-04T14:58:52,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/21b2d193514a40fba209310da4c2450b, entries=150, sequenceid=363, filesize=12.0 K 2024-12-04T14:58:52,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e727f8587f46eb6f1b53c466f4689322 in 1286ms, sequenceid=363, compaction requested=false 2024-12-04T14:58:52,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:52,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34477 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T14:58:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:52,492 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-04T14:58:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204dcf1a1318eda421786a7d22dcc62eb3e_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324331181/Put/seqid=0 2024-12-04T14:58:52,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742398_1574 (size=12454) 2024-12-04T14:58:52,581 INFO [Thread-2142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2142. 2024-12-04T14:58:52,581 DEBUG [Thread-2142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:52,581 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,581 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:52,581 INFO [Thread-2140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2140. 2024-12-04T14:58:52,581 DEBUG [Thread-2140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:52,581 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,581 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:52,582 INFO [Thread-2144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2144. 2024-12-04T14:58:52,582 DEBUG [Thread-2144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:52,582 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,582 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:52,583 INFO [Thread-2138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2138. 2024-12-04T14:58:52,583 DEBUG [Thread-2138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:52,583 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,583 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:52,583 INFO [Thread-2136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2136. 2024-12-04T14:58:52,583 DEBUG [Thread-2136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:52,583 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,583 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:52,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T14:58:52,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:58:52,907 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204dcf1a1318eda421786a7d22dcc62eb3e_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204dcf1a1318eda421786a7d22dcc62eb3e_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/4728fbfd42f44786836b53d1bc64b641, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:58:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/4728fbfd42f44786836b53d1bc64b641 is 175, key is test_row_0/A:col10/1733324331181/Put/seqid=0 2024-12-04T14:58:52,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742399_1575 (size=31255) 2024-12-04T14:58:53,315 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/4728fbfd42f44786836b53d1bc64b641 2024-12-04T14:58:53,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/cfe922fdc5df4330ae669ddde3929d2b is 50, key is test_row_0/B:col10/1733324331181/Put/seqid=0 2024-12-04T14:58:53,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742400_1576 (size=12301) 2024-12-04T14:58:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34477 {}] regionserver.HRegion(8855): Flush requested on e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:58:53,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. as already flushing 2024-12-04T14:58:53,346 INFO [Thread-2127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2127. 2024-12-04T14:58:53,346 DEBUG [Thread-2127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:58:53,346 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:53,346 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:58:53,346 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:58:53,724 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/cfe922fdc5df4330ae669ddde3929d2b 2024-12-04T14:58:53,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/1ae79db156b445ea93b5cb715566042d is 50, key is test_row_0/C:col10/1733324331181/Put/seqid=0 2024-12-04T14:58:53,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742401_1577 (size=12301) 2024-12-04T14:58:54,141 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/1ae79db156b445ea93b5cb715566042d 2024-12-04T14:58:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/4728fbfd42f44786836b53d1bc64b641 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/4728fbfd42f44786836b53d1bc64b641 2024-12-04T14:58:54,151 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/4728fbfd42f44786836b53d1bc64b641, entries=150, sequenceid=391, filesize=30.5 K 2024-12-04T14:58:54,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/cfe922fdc5df4330ae669ddde3929d2b as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/cfe922fdc5df4330ae669ddde3929d2b 2024-12-04T14:58:54,156 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/cfe922fdc5df4330ae669ddde3929d2b, entries=150, sequenceid=391, filesize=12.0 K 2024-12-04T14:58:54,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/1ae79db156b445ea93b5cb715566042d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/1ae79db156b445ea93b5cb715566042d 2024-12-04T14:58:54,160 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/1ae79db156b445ea93b5cb715566042d, entries=150, sequenceid=391, filesize=12.0 K 2024-12-04T14:58:54,161 INFO [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=6.71 KB/6870 for e727f8587f46eb6f1b53c466f4689322 in 1669ms, sequenceid=391, compaction requested=true 2024-12-04T14:58:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for e727f8587f46eb6f1b53c466f4689322: 2024-12-04T14:58:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:58:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d214edae7bd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-04T14:58:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-04T14:58:54,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-04T14:58:54,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5030 sec 2024-12-04T14:58:54,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.5080 sec 2024-12-04T14:58:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T14:58:54,816 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:58:56,578 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T14:59:00,916 INFO [Thread-2125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2125. 2024-12-04T14:59:00,916 DEBUG [Thread-2125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:00,916 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,917 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:00,980 INFO [Thread-2131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2131. 2024-12-04T14:59:00,980 DEBUG [Thread-2131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:00,980 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,980 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:00,987 INFO [Thread-2133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2133. 2024-12-04T14:59:00,987 DEBUG [Thread-2133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:00,987 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,987 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:00,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:01,026 INFO [Thread-2129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2129. 2024-12-04T14:59:01,026 DEBUG [Thread-2129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T14:59:01,026 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:01,026 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:01,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 4 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 287 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 5 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3257 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3230 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3216 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3234 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3190 2024-12-04T14:59:01,027 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T14:59:01,027 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:59:01,027 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:59:01,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:01,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:01,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:01,029 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T14:59:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T14:59:01,032 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324341032"}]},"ts":"1733324341032"} 2024-12-04T14:59:01,034 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T14:59:01,034 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T14:59:01,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T14:59:01,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, UNASSIGN}] 2024-12-04T14:59:01,038 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, UNASSIGN 2024-12-04T14:59:01,038 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=e727f8587f46eb6f1b53c466f4689322, regionState=CLOSING, regionLocation=d214edae7bd2,34477,1733324158096 2024-12-04T14:59:01,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, UNASSIGN because future has completed 2024-12-04T14:59:01,040 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T14:59:01,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096}] 2024-12-04T14:59:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T14:59:01,197 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,197 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T14:59:01,197 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing e727f8587f46eb6f1b53c466f4689322, disabling compactions & flushes 2024-12-04T14:59:01,197 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:59:01,197 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:59:01,197 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. after waiting 0 ms 2024-12-04T14:59:01,197 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:59:01,197 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing e727f8587f46eb6f1b53c466f4689322 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-04T14:59:01,212 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120418bde1f0d7884c1ba9975cbbabd0316f_e727f8587f46eb6f1b53c466f4689322 is 50, key is test_row_0/A:col10/1733324340985/Put/seqid=0 2024-12-04T14:59:01,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742402_1578 (size=9914) 2024-12-04T14:59:01,221 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T14:59:01,234 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120418bde1f0d7884c1ba9975cbbabd0316f_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120418bde1f0d7884c1ba9975cbbabd0316f_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,235 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/4b067b6eef124eb7a1609efff216584f, store: [table=TestAcidGuarantees family=A region=e727f8587f46eb6f1b53c466f4689322] 2024-12-04T14:59:01,236 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/4b067b6eef124eb7a1609efff216584f is 175, key is test_row_0/A:col10/1733324340985/Put/seqid=0 2024-12-04T14:59:01,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742403_1579 (size=22561) 2024-12-04T14:59:01,254 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/4b067b6eef124eb7a1609efff216584f 2024-12-04T14:59:01,272 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/1b6015fdb5e246f287ad2ce0f521b27d is 50, key is test_row_0/B:col10/1733324340985/Put/seqid=0 2024-12-04T14:59:01,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742404_1580 (size=9857) 2024-12-04T14:59:01,286 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/1b6015fdb5e246f287ad2ce0f521b27d 2024-12-04T14:59:01,291 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/9090a36255ba481ca71e202e5c56de84 is 50, key is test_row_0/C:col10/1733324340985/Put/seqid=0 2024-12-04T14:59:01,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742405_1581 (size=9857) 2024-12-04T14:59:01,308 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/9090a36255ba481ca71e202e5c56de84 2024-12-04T14:59:01,316 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/A/4b067b6eef124eb7a1609efff216584f as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/4b067b6eef124eb7a1609efff216584f 2024-12-04T14:59:01,320 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/4b067b6eef124eb7a1609efff216584f, entries=100, sequenceid=399, filesize=22.0 K 2024-12-04T14:59:01,320 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/B/1b6015fdb5e246f287ad2ce0f521b27d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/1b6015fdb5e246f287ad2ce0f521b27d 2024-12-04T14:59:01,323 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/1b6015fdb5e246f287ad2ce0f521b27d, entries=100, sequenceid=399, filesize=9.6 K 2024-12-04T14:59:01,331 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/.tmp/C/9090a36255ba481ca71e202e5c56de84 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/9090a36255ba481ca71e202e5c56de84 2024-12-04T14:59:01,338 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/9090a36255ba481ca71e202e5c56de84, entries=100, sequenceid=399, filesize=9.6 K 2024-12-04T14:59:01,340 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e727f8587f46eb6f1b53c466f4689322 in 143ms, sequenceid=399, compaction requested=true 2024-12-04T14:59:01,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/737551c1feb7488ea352b16c9c0166fe, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/be3eebe54f854b1eb96fd1b08605dc38, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7bf55ef5d0604b27bab22000cf1f8ab5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d673e433dfa54122bd388825b58f2597, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0604a6042b6048d8b1723914c0ebb599, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d79da3cac2bf47a688d73136ff9e9b25, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/96578f97103948718bdf560a9b839301, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cbbafac124cb405994f73468217d2d80, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cb52c3bfe63548dab5389def63988e44, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cfa5c427ddc14b50b3ffba9e488db742, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/25113c5e8f434282a31da11823e6eec8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cc2994fa05c841b2b73d56643555380e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/52f27b6d8d1f45a3b3aaa1c946a076e2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7c4d964f9dd94b18898ba6e71eb18346, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0f93e09c82564d2e93a4adffad21af6e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/a40357a421c647c8b6cc6090285e8dfe, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/6e2ca07774f2449ba3c653bc23c52595, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8db77511d23749598a6bdd4fcdf97e53, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/fd805af59c9643dca471865ebd443a59, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/51642eccdb5c4dacb7da83e8ff8f169d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/5a8bb5b413764e939237aa613fe16899, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/31a9ab15520e468dbb40e57f7310337f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0e1c339f73cd4bd3b9313c846a56c1e1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8d2a460c156f4ec197c29668d6099d32] to archive 2024-12-04T14:59:01,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:59:01,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/737551c1feb7488ea352b16c9c0166fe to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/737551c1feb7488ea352b16c9c0166fe 2024-12-04T14:59:01,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/be3eebe54f854b1eb96fd1b08605dc38 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/be3eebe54f854b1eb96fd1b08605dc38 2024-12-04T14:59:01,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7bf55ef5d0604b27bab22000cf1f8ab5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7bf55ef5d0604b27bab22000cf1f8ab5 2024-12-04T14:59:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T14:59:01,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d673e433dfa54122bd388825b58f2597 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d673e433dfa54122bd388825b58f2597 2024-12-04T14:59:01,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0604a6042b6048d8b1723914c0ebb599 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0604a6042b6048d8b1723914c0ebb599 2024-12-04T14:59:01,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d79da3cac2bf47a688d73136ff9e9b25 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/d79da3cac2bf47a688d73136ff9e9b25 2024-12-04T14:59:01,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/96578f97103948718bdf560a9b839301 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/96578f97103948718bdf560a9b839301 2024-12-04T14:59:01,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cbbafac124cb405994f73468217d2d80 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cbbafac124cb405994f73468217d2d80 2024-12-04T14:59:01,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cb52c3bfe63548dab5389def63988e44 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cb52c3bfe63548dab5389def63988e44 2024-12-04T14:59:01,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cfa5c427ddc14b50b3ffba9e488db742 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cfa5c427ddc14b50b3ffba9e488db742 2024-12-04T14:59:01,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/25113c5e8f434282a31da11823e6eec8 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/25113c5e8f434282a31da11823e6eec8 2024-12-04T14:59:01,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cc2994fa05c841b2b73d56643555380e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/cc2994fa05c841b2b73d56643555380e 2024-12-04T14:59:01,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/52f27b6d8d1f45a3b3aaa1c946a076e2 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/52f27b6d8d1f45a3b3aaa1c946a076e2 2024-12-04T14:59:01,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7c4d964f9dd94b18898ba6e71eb18346 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/7c4d964f9dd94b18898ba6e71eb18346 2024-12-04T14:59:01,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0f93e09c82564d2e93a4adffad21af6e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0f93e09c82564d2e93a4adffad21af6e 2024-12-04T14:59:01,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/a40357a421c647c8b6cc6090285e8dfe to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/a40357a421c647c8b6cc6090285e8dfe 2024-12-04T14:59:01,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/6e2ca07774f2449ba3c653bc23c52595 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/6e2ca07774f2449ba3c653bc23c52595 2024-12-04T14:59:01,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8db77511d23749598a6bdd4fcdf97e53 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8db77511d23749598a6bdd4fcdf97e53 2024-12-04T14:59:01,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/fd805af59c9643dca471865ebd443a59 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/fd805af59c9643dca471865ebd443a59 2024-12-04T14:59:01,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/51642eccdb5c4dacb7da83e8ff8f169d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/51642eccdb5c4dacb7da83e8ff8f169d 2024-12-04T14:59:01,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/5a8bb5b413764e939237aa613fe16899 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/5a8bb5b413764e939237aa613fe16899 2024-12-04T14:59:01,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/31a9ab15520e468dbb40e57f7310337f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/31a9ab15520e468dbb40e57f7310337f 2024-12-04T14:59:01,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0e1c339f73cd4bd3b9313c846a56c1e1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/0e1c339f73cd4bd3b9313c846a56c1e1 2024-12-04T14:59:01,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8d2a460c156f4ec197c29668d6099d32 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/8d2a460c156f4ec197c29668d6099d32 2024-12-04T14:59:01,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b6d94b65d83545efa856c01facc2d9bd, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b78a8d52f3284cc397cd8567900c66af, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/a881e8f188294e7d884c45ab99832cdc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/6546f881276447868fc5ed469882253e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b33b6be5d9a4460d8aeed43ad06b718a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/401302e0fc734fe9a39187aa49d2a96d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/14ec43d00a044fd98fa6272bde24e07b, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8261d2db54614a9baa7a527db8e1251f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/08c943cef3224a1d823b7cda8c22d2ac, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/bd4c0698f35045e8bc38ef7743cb999c, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/78279921a05548079a73922f2724c093, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8d2c350a33c2415fb1d788390fd9d3c8, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/153f9a97c3da421dbd59f8074178919d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/78699dec8aef45288a2ad7251d88bd12, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/d63555ee20244b25a7fd4fe408dbf8ea, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b13d965a7bdc4545a98ac2b8f4f9f2fa, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/e58c779e688a42b6b08424d5d5306514, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/6df48f929cad45a1a2485cb8d669ba22, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/85383e27fdac4209aa40d4cba477c652, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/dc5207fadafc46ab80949aa98e8060eb, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/c78ad3b1c4ce4a78a8a413ce032433c4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/76fd0889a9ce458492f75964d439ec2f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/20fc97f5d4fc4ccf95885ae416b0cea4, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/4091337f35b74dd583b9b50ca8ea40d5] to archive 2024-12-04T14:59:01,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:59:01,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b6d94b65d83545efa856c01facc2d9bd to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b6d94b65d83545efa856c01facc2d9bd 2024-12-04T14:59:01,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b78a8d52f3284cc397cd8567900c66af to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b78a8d52f3284cc397cd8567900c66af 2024-12-04T14:59:01,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/a881e8f188294e7d884c45ab99832cdc to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/a881e8f188294e7d884c45ab99832cdc 2024-12-04T14:59:01,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/6546f881276447868fc5ed469882253e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/6546f881276447868fc5ed469882253e 2024-12-04T14:59:01,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b33b6be5d9a4460d8aeed43ad06b718a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b33b6be5d9a4460d8aeed43ad06b718a 2024-12-04T14:59:01,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/401302e0fc734fe9a39187aa49d2a96d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/401302e0fc734fe9a39187aa49d2a96d 2024-12-04T14:59:01,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/14ec43d00a044fd98fa6272bde24e07b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/14ec43d00a044fd98fa6272bde24e07b 2024-12-04T14:59:01,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8261d2db54614a9baa7a527db8e1251f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8261d2db54614a9baa7a527db8e1251f 2024-12-04T14:59:01,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/08c943cef3224a1d823b7cda8c22d2ac to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/08c943cef3224a1d823b7cda8c22d2ac 2024-12-04T14:59:01,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/bd4c0698f35045e8bc38ef7743cb999c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/bd4c0698f35045e8bc38ef7743cb999c 2024-12-04T14:59:01,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/78279921a05548079a73922f2724c093 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/78279921a05548079a73922f2724c093 2024-12-04T14:59:01,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8d2c350a33c2415fb1d788390fd9d3c8 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/8d2c350a33c2415fb1d788390fd9d3c8 2024-12-04T14:59:01,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/153f9a97c3da421dbd59f8074178919d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/153f9a97c3da421dbd59f8074178919d 2024-12-04T14:59:01,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/78699dec8aef45288a2ad7251d88bd12 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/78699dec8aef45288a2ad7251d88bd12 2024-12-04T14:59:01,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/d63555ee20244b25a7fd4fe408dbf8ea to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/d63555ee20244b25a7fd4fe408dbf8ea 2024-12-04T14:59:01,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b13d965a7bdc4545a98ac2b8f4f9f2fa to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b13d965a7bdc4545a98ac2b8f4f9f2fa 2024-12-04T14:59:01,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/e58c779e688a42b6b08424d5d5306514 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/e58c779e688a42b6b08424d5d5306514 2024-12-04T14:59:01,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/6df48f929cad45a1a2485cb8d669ba22 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/6df48f929cad45a1a2485cb8d669ba22 2024-12-04T14:59:01,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/85383e27fdac4209aa40d4cba477c652 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/85383e27fdac4209aa40d4cba477c652 2024-12-04T14:59:01,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/dc5207fadafc46ab80949aa98e8060eb to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/dc5207fadafc46ab80949aa98e8060eb 2024-12-04T14:59:01,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/c78ad3b1c4ce4a78a8a413ce032433c4 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/c78ad3b1c4ce4a78a8a413ce032433c4 2024-12-04T14:59:01,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/76fd0889a9ce458492f75964d439ec2f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/76fd0889a9ce458492f75964d439ec2f 2024-12-04T14:59:01,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/20fc97f5d4fc4ccf95885ae416b0cea4 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/20fc97f5d4fc4ccf95885ae416b0cea4 2024-12-04T14:59:01,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/4091337f35b74dd583b9b50ca8ea40d5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/4091337f35b74dd583b9b50ca8ea40d5 2024-12-04T14:59:01,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2b7ff23bcb6a4d319b9b7ed98f7932bb, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/9d1ca03c26b841749c8e4a1b9f767ebe, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/e616b3fb3e204d01beb2cfa76dbb37df, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/5aab64e274bf4d45a4a087e08f7fa881, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/31f052cd28e141e4983f1300d5e2f17d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2696d07bb2dc457d8a24b0008c5652c1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/6d2352b3556049c081f90b21f3d1cdd1, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/7027cc679b8d4d9b882e8faa436d571a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/eec8137bba69489483ee27ff6fceaec2, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/11ea3e9084c14543b7af448518045224, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/7521b52d9106411aa4f64ea857e2ca1e, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/4e97adc49b5942018950f888ca13608d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/06c593e4894743ec8d04f15c2bfd59c3, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b8490b2d2bc84bb6a2ae59b2884b3d5d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/0173bf2a88da475da71559747b98c7f7, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/28ce659403a64434af7efb432dd4f4fc, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2e823d35633e4073ba16be3b3c0b9ee5, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/3e5b690ee4fd4767a1de956d11da685f, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/c300c249b7424f4fb501938466b76026, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/bb3d1a6039f84261b587c8eedbb1dc0d, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/5c69953ff4334f939960266a7ad43f44, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b79304bc3ff6415faf1eeda5ef4ccf30, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b3484ccf4c844ad194a7b01b0da62f6a, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/89e9371a91aa4274a5094c98d447227c] to archive 2024-12-04T14:59:01,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T14:59:01,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2b7ff23bcb6a4d319b9b7ed98f7932bb to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2b7ff23bcb6a4d319b9b7ed98f7932bb 2024-12-04T14:59:01,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/9d1ca03c26b841749c8e4a1b9f767ebe to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/9d1ca03c26b841749c8e4a1b9f767ebe 2024-12-04T14:59:01,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/e616b3fb3e204d01beb2cfa76dbb37df to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/e616b3fb3e204d01beb2cfa76dbb37df 2024-12-04T14:59:01,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/5aab64e274bf4d45a4a087e08f7fa881 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/5aab64e274bf4d45a4a087e08f7fa881 2024-12-04T14:59:01,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/31f052cd28e141e4983f1300d5e2f17d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/31f052cd28e141e4983f1300d5e2f17d 2024-12-04T14:59:01,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2696d07bb2dc457d8a24b0008c5652c1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2696d07bb2dc457d8a24b0008c5652c1 2024-12-04T14:59:01,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/6d2352b3556049c081f90b21f3d1cdd1 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/6d2352b3556049c081f90b21f3d1cdd1 2024-12-04T14:59:01,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/7027cc679b8d4d9b882e8faa436d571a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/7027cc679b8d4d9b882e8faa436d571a 2024-12-04T14:59:01,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/eec8137bba69489483ee27ff6fceaec2 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/eec8137bba69489483ee27ff6fceaec2 2024-12-04T14:59:01,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/11ea3e9084c14543b7af448518045224 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/11ea3e9084c14543b7af448518045224 2024-12-04T14:59:01,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/7521b52d9106411aa4f64ea857e2ca1e to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/7521b52d9106411aa4f64ea857e2ca1e 2024-12-04T14:59:01,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/4e97adc49b5942018950f888ca13608d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/4e97adc49b5942018950f888ca13608d 2024-12-04T14:59:01,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/06c593e4894743ec8d04f15c2bfd59c3 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/06c593e4894743ec8d04f15c2bfd59c3 2024-12-04T14:59:01,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b8490b2d2bc84bb6a2ae59b2884b3d5d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b8490b2d2bc84bb6a2ae59b2884b3d5d 2024-12-04T14:59:01,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/0173bf2a88da475da71559747b98c7f7 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/0173bf2a88da475da71559747b98c7f7 2024-12-04T14:59:01,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/28ce659403a64434af7efb432dd4f4fc to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/28ce659403a64434af7efb432dd4f4fc 2024-12-04T14:59:01,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2e823d35633e4073ba16be3b3c0b9ee5 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/2e823d35633e4073ba16be3b3c0b9ee5 2024-12-04T14:59:01,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/3e5b690ee4fd4767a1de956d11da685f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/3e5b690ee4fd4767a1de956d11da685f 2024-12-04T14:59:01,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/c300c249b7424f4fb501938466b76026 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/c300c249b7424f4fb501938466b76026 2024-12-04T14:59:01,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/bb3d1a6039f84261b587c8eedbb1dc0d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/bb3d1a6039f84261b587c8eedbb1dc0d 2024-12-04T14:59:01,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/5c69953ff4334f939960266a7ad43f44 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/5c69953ff4334f939960266a7ad43f44 2024-12-04T14:59:01,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b79304bc3ff6415faf1eeda5ef4ccf30 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b79304bc3ff6415faf1eeda5ef4ccf30 2024-12-04T14:59:01,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b3484ccf4c844ad194a7b01b0da62f6a to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/b3484ccf4c844ad194a7b01b0da62f6a 2024-12-04T14:59:01,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/89e9371a91aa4274a5094c98d447227c to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/89e9371a91aa4274a5094c98d447227c 2024-12-04T14:59:01,510 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/recovered.edits/402.seqid, newMaxSeqId=402, maxSeqId=4 2024-12-04T14:59:01,510 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. 2024-12-04T14:59:01,511 DEBUG [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for e727f8587f46eb6f1b53c466f4689322: Waiting for close lock at 1733324341197Running coprocessor pre-close hooks at 1733324341197Disabling compacts and flushes for region at 1733324341197Disabling writes for close at 1733324341197Obtaining lock to block concurrent updates at 1733324341197Preparing flush snapshotting stores in e727f8587f46eb6f1b53c466f4689322 at 1733324341197Finished memstore snapshotting TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733324341198 (+1 ms)Flushing stores of TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322. at 1733324341200 (+2 ms)Flushing e727f8587f46eb6f1b53c466f4689322/A: creating writer at 1733324341200Flushing mob file e727f8587f46eb6f1b53c466f4689322/A: appending metadata at 1733324341212 (+12 ms)Flushing mob file e727f8587f46eb6f1b53c466f4689322/A: closing flushed file at 1733324341212Flushing e727f8587f46eb6f1b53c466f4689322/A: appending metadata at 1733324341235 (+23 ms)Flushing e727f8587f46eb6f1b53c466f4689322/A: closing flushed file at 1733324341235Flushing e727f8587f46eb6f1b53c466f4689322/B: creating writer at 1733324341260 (+25 ms)Flushing e727f8587f46eb6f1b53c466f4689322/B: appending metadata at 1733324341271 (+11 ms)Flushing e727f8587f46eb6f1b53c466f4689322/B: closing flushed file at 1733324341271Flushing e727f8587f46eb6f1b53c466f4689322/C: creating writer at 1733324341289 (+18 ms)Flushing e727f8587f46eb6f1b53c466f4689322/C: appending metadata at 1733324341290 (+1 ms)Flushing e727f8587f46eb6f1b53c466f4689322/C: closing flushed file at 1733324341290Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e94be67: reopening flushed file at 1733324341314 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b990d4b: reopening flushed file at 1733324341320 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35c181e2: reopening flushed file at 1733324341324 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e727f8587f46eb6f1b53c466f4689322 in 143ms, sequenceid=399, compaction requested=true at 1733324341340 (+16 ms)Writing region close event to WAL at 1733324341506 (+166 ms)Running coprocessor post-close hooks at 1733324341510 (+4 ms)Closed at 1733324341510 2024-12-04T14:59:01,513 INFO [RS_CLOSE_REGION-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,513 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=e727f8587f46eb6f1b53c466f4689322, regionState=CLOSED 2024-12-04T14:59:01,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 because future has completed 2024-12-04T14:59:01,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-04T14:59:01,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure e727f8587f46eb6f1b53c466f4689322, server=d214edae7bd2,34477,1733324158096 in 484 msec 2024-12-04T14:59:01,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-12-04T14:59:01,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e727f8587f46eb6f1b53c466f4689322, UNASSIGN in 489 msec 2024-12-04T14:59:01,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-04T14:59:01,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-12-04T14:59:01,534 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733324341534"}]},"ts":"1733324341534"} 2024-12-04T14:59:01,536 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T14:59:01,536 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T14:59:01,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 509 msec 2024-12-04T14:59:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T14:59:01,656 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:01,657 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T14:59:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:01,660 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:01,660 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:01,662 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T14:59:01,666 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C, FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/recovered.edits] 2024-12-04T14:59:01,678 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/26a8b23c0aae4827b29fc91a8caaba38 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/26a8b23c0aae4827b29fc91a8caaba38 2024-12-04T14:59:01,684 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/4728fbfd42f44786836b53d1bc64b641 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/4728fbfd42f44786836b53d1bc64b641 2024-12-04T14:59:01,687 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/4b067b6eef124eb7a1609efff216584f to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/4b067b6eef124eb7a1609efff216584f 2024-12-04T14:59:01,692 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/f301014e07934730a11387c0b64661d6 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/A/f301014e07934730a11387c0b64661d6 2024-12-04T14:59:01,696 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/1b6015fdb5e246f287ad2ce0f521b27d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/1b6015fdb5e246f287ad2ce0f521b27d 2024-12-04T14:59:01,697 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/92ba2118d00343729551ea5139ba0546 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/92ba2118d00343729551ea5139ba0546 2024-12-04T14:59:01,698 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b92c08e294724e768d5a88b1c28a741d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/b92c08e294724e768d5a88b1c28a741d 2024-12-04T14:59:01,702 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/cfe922fdc5df4330ae669ddde3929d2b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/B/cfe922fdc5df4330ae669ddde3929d2b 2024-12-04T14:59:01,705 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/1ae79db156b445ea93b5cb715566042d to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/1ae79db156b445ea93b5cb715566042d 2024-12-04T14:59:01,706 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/21b2d193514a40fba209310da4c2450b to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/21b2d193514a40fba209310da4c2450b 2024-12-04T14:59:01,711 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/385906e9edcc4423a4e754330a5090fe to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/385906e9edcc4423a4e754330a5090fe 2024-12-04T14:59:01,713 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/9090a36255ba481ca71e202e5c56de84 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/C/9090a36255ba481ca71e202e5c56de84 2024-12-04T14:59:01,716 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/recovered.edits/402.seqid to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322/recovered.edits/402.seqid 2024-12-04T14:59:01,717 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/default/TestAcidGuarantees/e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,717 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T14:59:01,717 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T14:59:01,718 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T14:59:01,720 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120410f57654685b468c944487c70ae9a94c_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120410f57654685b468c944487c70ae9a94c_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,721 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204112b3121a6d64377a177982ea1ca10c3_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204112b3121a6d64377a177982ea1ca10c3_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,722 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120418bde1f0d7884c1ba9975cbbabd0316f_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120418bde1f0d7884c1ba9975cbbabd0316f_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,722 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041cf89d957f944d7682b87ec2134e2e21_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412041cf89d957f944d7682b87ec2134e2e21_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,724 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042889e3e5b2244c5c888ce6ce002ffbff_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042889e3e5b2244c5c888ce6ce002ffbff_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,725 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120439285baa41d24ee6b40e05b3dc8e326b_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120439285baa41d24ee6b40e05b3dc8e326b_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,727 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204525dd10927c54c68b88c93899e99e33f_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204525dd10927c54c68b88c93899e99e33f_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,731 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120458d0856d05084de0aaa70172a8ff6a3a_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120458d0856d05084de0aaa70172a8ff6a3a_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,733 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120458e1527ab0664b1380445573b2052e32_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120458e1527ab0664b1380445573b2052e32_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,734 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120471a273af36464f44947db40e03766d7a_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120471a273af36464f44947db40e03766d7a_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,736 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047e779856a8b24c3cb7d6f92a9f3329f2_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047e779856a8b24c3cb7d6f92a9f3329f2_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,737 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048481e9ec75554a79a4e73110834ba92a_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048481e9ec75554a79a4e73110834ba92a_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,742 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048662466828c74cd7af5bcdcb68f5d27c_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048662466828c74cd7af5bcdcb68f5d27c_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,743 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048a121e7f42e440ce95a691f773c928ce_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412048a121e7f42e440ce95a691f773c928ce_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,745 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b91ab7c5d19a4dd59e4873410977ccd9_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b91ab7c5d19a4dd59e4873410977ccd9_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,747 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204be90b16237034bf5bea02341ff5c81bf_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204be90b16237034bf5bea02341ff5c81bf_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,752 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d4a5f2ca2a16487db16b44f78c837a0b_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d4a5f2ca2a16487db16b44f78c837a0b_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,753 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d8bd1b85095749448fda15eca075837d_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d8bd1b85095749448fda15eca075837d_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,756 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204dcf1a1318eda421786a7d22dcc62eb3e_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204dcf1a1318eda421786a7d22dcc62eb3e_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,757 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f72480d7158a4d6a82aefbc8b89ab9f2_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204f72480d7158a4d6a82aefbc8b89ab9f2_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,759 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204fb6a6157fe3a4b4ea8eda76033fe0baa_e727f8587f46eb6f1b53c466f4689322 to hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204fb6a6157fe3a4b4ea8eda76033fe0baa_e727f8587f46eb6f1b53c466f4689322 2024-12-04T14:59:01,759 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T14:59:01,765 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:01,768 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T14:59:01,771 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T14:59:01,772 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:01,772 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T14:59:01,772 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733324341772"}]},"ts":"9223372036854775807"} 2024-12-04T14:59:01,775 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T14:59:01,775 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e727f8587f46eb6f1b53c466f4689322, NAME => 'TestAcidGuarantees,,1733324309720.e727f8587f46eb6f1b53c466f4689322.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T14:59:01,775 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T14:59:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T14:59:01,775 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733324341775"}]},"ts":"9223372036854775807"} 2024-12-04T14:59:01,778 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T14:59:01,779 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T14:59:01,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 122 msec 2024-12-04T14:59:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45797 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T14:59:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T14:59:01,986 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T14:59:01,998 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 236) - Thread LEAK? -, OpenFileDescriptor=443 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=632 (was 473) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8883 (was 9443) 2024-12-04T14:59:01,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T14:59:01,998 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:59:01,999 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:59:01,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:01,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:01,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T14:59:01,999 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T14:59:01,999 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=900566320, stopped=false 2024-12-04T14:59:02,000 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d214edae7bd2,45797,1733324157426 2024-12-04T14:59:02,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:59:02,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T14:59:02,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:59:02,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T14:59:02,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:59:02,051 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T14:59:02,051 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:59:02,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:02,052 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd214edae7bd2,34477,1733324158096' ***** 2024-12-04T14:59:02,052 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T14:59:02,053 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T14:59:02,053 INFO [RS:0;d214edae7bd2:34477 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T14:59:02,053 INFO [RS:0;d214edae7bd2:34477 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T14:59:02,053 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(959): stopping server d214edae7bd2,34477,1733324158096 2024-12-04T14:59:02,053 INFO [RS:0;d214edae7bd2:34477 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T14:59:02,053 INFO [RS:0;d214edae7bd2:34477 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d214edae7bd2:34477. 2024-12-04T14:59:02,053 DEBUG [RS:0;d214edae7bd2:34477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T14:59:02,053 DEBUG [RS:0;d214edae7bd2:34477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:02,054 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T14:59:02,054 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T14:59:02,054 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T14:59:02,054 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T14:59:02,054 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T14:59:02,055 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:59:02,055 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T14:59:02,055 DEBUG [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T14:59:02,056 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T14:59:02,056 DEBUG [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T14:59:02,057 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T14:59:02,057 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T14:59:02,057 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T14:59:02,057 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T14:59:02,057 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T14:59:02,057 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-04T14:59:02,096 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/info/412eea1667ba456897b18f3caced83d3 is 95, key is TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1./info:/1733324187485/DeleteFamily/seqid=0 2024-12-04T14:59:02,097 INFO [regionserver/d214edae7bd2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:59:02,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742406_1582 (size=6514) 2024-12-04T14:59:02,105 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/info/412eea1667ba456897b18f3caced83d3 2024-12-04T14:59:02,145 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/ns/02f41cba27f8495ea1d9c27bcec80768 is 93, key is TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1./ns:/1733324187449/DeleteFamily/seqid=0 2024-12-04T14:59:02,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742407_1583 (size=5906) 2024-12-04T14:59:02,161 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/ns/02f41cba27f8495ea1d9c27bcec80768 2024-12-04T14:59:02,188 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/rep_barrier/aad066dffc784e7b899187c84b560fa5 is 102, key is TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1./rep_barrier:/1733324187449/DeleteFamily/seqid=0 2024-12-04T14:59:02,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742408_1584 (size=6025) 2024-12-04T14:59:02,201 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/rep_barrier/aad066dffc784e7b899187c84b560fa5 2024-12-04T14:59:02,233 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/table/a4bb45a54bc846189d0249c6c558a8e3 is 96, key is TestAcidGuarantees,,1733324161281.7618a4771c68a9e87db105604d8452e1./table:/1733324187449/DeleteFamily/seqid=0 2024-12-04T14:59:02,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742409_1585 (size=5936) 2024-12-04T14:59:02,255 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/table/a4bb45a54bc846189d0249c6c558a8e3 2024-12-04T14:59:02,256 DEBUG [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T14:59:02,260 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/info/412eea1667ba456897b18f3caced83d3 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/info/412eea1667ba456897b18f3caced83d3 2024-12-04T14:59:02,265 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/info/412eea1667ba456897b18f3caced83d3, entries=12, sequenceid=90, filesize=6.4 K 2024-12-04T14:59:02,266 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/ns/02f41cba27f8495ea1d9c27bcec80768 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/ns/02f41cba27f8495ea1d9c27bcec80768 2024-12-04T14:59:02,271 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/ns/02f41cba27f8495ea1d9c27bcec80768, entries=8, sequenceid=90, filesize=5.8 K 2024-12-04T14:59:02,272 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/rep_barrier/aad066dffc784e7b899187c84b560fa5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/rep_barrier/aad066dffc784e7b899187c84b560fa5 2024-12-04T14:59:02,283 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/rep_barrier/aad066dffc784e7b899187c84b560fa5, entries=6, sequenceid=90, filesize=5.9 K 2024-12-04T14:59:02,284 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/.tmp/table/a4bb45a54bc846189d0249c6c558a8e3 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/table/a4bb45a54bc846189d0249c6c558a8e3 2024-12-04T14:59:02,290 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/table/a4bb45a54bc846189d0249c6c558a8e3, entries=7, sequenceid=90, filesize=5.8 K 2024-12-04T14:59:02,292 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 234ms, sequenceid=90, compaction requested=false 2024-12-04T14:59:02,359 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-04T14:59:02,362 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T14:59:02,362 INFO [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T14:59:02,362 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733324342056Running coprocessor pre-close hooks at 1733324342057 (+1 ms)Disabling compacts and flushes for region at 1733324342057Disabling writes for close at 1733324342057Obtaining lock to block concurrent updates at 1733324342057Preparing flush snapshotting stores in 1588230740 at 1733324342057Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733324342057Flushing stores of hbase:meta,,1.1588230740 at 1733324342066 (+9 ms)Flushing 1588230740/info: creating writer at 1733324342067 (+1 ms)Flushing 1588230740/info: appending metadata at 1733324342095 (+28 ms)Flushing 1588230740/info: closing flushed file at 1733324342095Flushing 1588230740/ns: creating writer at 1733324342113 (+18 ms)Flushing 1588230740/ns: appending metadata at 1733324342144 (+31 ms)Flushing 1588230740/ns: closing flushed file at 1733324342144Flushing 1588230740/rep_barrier: creating writer at 1733324342166 (+22 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733324342188 (+22 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733324342188Flushing 1588230740/table: creating writer at 1733324342206 (+18 ms)Flushing 1588230740/table: appending metadata at 1733324342232 (+26 ms)Flushing 1588230740/table: closing flushed file at 1733324342232Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40b4c600: reopening flushed file at 1733324342260 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b651ec: reopening flushed file at 1733324342265 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fa79aa9: reopening flushed file at 1733324342271 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bbed5c6: reopening flushed file at 1733324342283 (+12 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 234ms, sequenceid=90, compaction requested=false at 1733324342292 (+9 ms)Writing region close event to WAL at 1733324342351 (+59 ms)Running coprocessor post-close hooks at 1733324342361 (+10 ms)Closed at 1733324342362 (+1 ms) 2024-12-04T14:59:02,362 DEBUG [RS_CLOSE_META-regionserver/d214edae7bd2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T14:59:02,456 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(976): stopping server d214edae7bd2,34477,1733324158096; all regions closed. 2024-12-04T14:59:02,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741834_1010 (size=25344) 2024-12-04T14:59:02,493 DEBUG [RS:0;d214edae7bd2:34477 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/oldWALs 2024-12-04T14:59:02,494 INFO [RS:0;d214edae7bd2:34477 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d214edae7bd2%2C34477%2C1733324158096.meta:.meta(num 1733324160864) 2024-12-04T14:59:02,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741832_1008 (size=13458534) 2024-12-04T14:59:02,523 DEBUG [RS:0;d214edae7bd2:34477 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/oldWALs 2024-12-04T14:59:02,523 INFO [RS:0;d214edae7bd2:34477 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d214edae7bd2%2C34477%2C1733324158096:(num 1733324159948) 2024-12-04T14:59:02,523 DEBUG [RS:0;d214edae7bd2:34477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T14:59:02,523 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T14:59:02,524 INFO [RS:0;d214edae7bd2:34477 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T14:59:02,524 INFO [RS:0;d214edae7bd2:34477 {}] hbase.ChoreService(370): Chore service for: regionserver/d214edae7bd2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T14:59:02,524 INFO [RS:0;d214edae7bd2:34477 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T14:59:02,525 INFO [RS:0;d214edae7bd2:34477 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34477 2024-12-04T14:59:02,527 INFO [regionserver/d214edae7bd2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T14:59:02,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d214edae7bd2,34477,1733324158096 2024-12-04T14:59:02,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T14:59:02,545 INFO [RS:0;d214edae7bd2:34477 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T14:59:02,545 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d214edae7bd2,34477,1733324158096] 2024-12-04T14:59:02,561 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d214edae7bd2,34477,1733324158096 already deleted, retry=false 2024-12-04T14:59:02,561 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d214edae7bd2,34477,1733324158096 expired; onlineServers=0 2024-12-04T14:59:02,561 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd214edae7bd2,45797,1733324157426' ***** 2024-12-04T14:59:02,561 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T14:59:02,562 INFO [M:0;d214edae7bd2:45797 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T14:59:02,562 INFO [M:0;d214edae7bd2:45797 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T14:59:02,562 DEBUG [M:0;d214edae7bd2:45797 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T14:59:02,562 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T14:59:02,562 DEBUG [M:0;d214edae7bd2:45797 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T14:59:02,562 DEBUG [master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.small.0-1733324159578 {}] cleaner.HFileCleaner(306): Exit Thread[master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.small.0-1733324159578,5,FailOnTimeoutGroup] 2024-12-04T14:59:02,562 INFO [M:0;d214edae7bd2:45797 {}] hbase.ChoreService(370): Chore service for: master/d214edae7bd2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T14:59:02,562 INFO [M:0;d214edae7bd2:45797 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T14:59:02,562 DEBUG [M:0;d214edae7bd2:45797 {}] master.HMaster(1795): Stopping service threads 2024-12-04T14:59:02,562 INFO [M:0;d214edae7bd2:45797 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T14:59:02,562 INFO [M:0;d214edae7bd2:45797 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T14:59:02,563 ERROR [M:0;d214edae7bd2:45797 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-04T14:59:02,563 INFO [M:0;d214edae7bd2:45797 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T14:59:02,563 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T14:59:02,562 DEBUG [master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.large.0-1733324159577 {}] cleaner.HFileCleaner(306): Exit Thread[master/d214edae7bd2:0:becomeActiveMaster-HFileCleaner.large.0-1733324159577,5,FailOnTimeoutGroup] 2024-12-04T14:59:02,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T14:59:02,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T14:59:02,570 DEBUG [M:0;d214edae7bd2:45797 {}] zookeeper.ZKUtil(347): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T14:59:02,570 WARN [M:0;d214edae7bd2:45797 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T14:59:02,572 INFO [M:0;d214edae7bd2:45797 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/.lastflushedseqids 2024-12-04T14:59:02,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742410_1586 (size=72) 2024-12-04T14:59:02,599 INFO [M:0;d214edae7bd2:45797 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T14:59:02,599 INFO [M:0;d214edae7bd2:45797 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T14:59:02,599 DEBUG [M:0;d214edae7bd2:45797 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T14:59:02,599 INFO [M:0;d214edae7bd2:45797 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:59:02,599 DEBUG [M:0;d214edae7bd2:45797 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:59:02,599 DEBUG [M:0;d214edae7bd2:45797 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T14:59:02,599 DEBUG [M:0;d214edae7bd2:45797 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:59:02,599 INFO [M:0;d214edae7bd2:45797 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=726.07 KB heapSize=890.51 KB 2024-12-04T14:59:02,625 DEBUG [M:0;d214edae7bd2:45797 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cfc80be2e9dc4bb5a720e21f101d60d5 is 82, key is hbase:meta,,1/info:regioninfo/1733324160941/Put/seqid=0 2024-12-04T14:59:02,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742411_1587 (size=5672) 2024-12-04T14:59:02,654 INFO [RS:0;d214edae7bd2:34477 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T14:59:02,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:59:02,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34477-0x101a22151460001, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:59:02,655 INFO [RS:0;d214edae7bd2:34477 {}] regionserver.HRegionServer(1031): Exiting; stopping=d214edae7bd2,34477,1733324158096; zookeeper connection closed. 2024-12-04T14:59:02,655 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@359d4d19 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@359d4d19 2024-12-04T14:59:02,655 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T14:59:03,033 INFO [M:0;d214edae7bd2:45797 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2022 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cfc80be2e9dc4bb5a720e21f101d60d5 2024-12-04T14:59:03,091 DEBUG [M:0;d214edae7bd2:45797 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1df0fe0a5c2e4fe2bed66e9f57b0fc3c is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x8B/proc:d/1733324312486/Put/seqid=0 2024-12-04T14:59:03,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742412_1588 (size=43138) 2024-12-04T14:59:03,120 INFO [M:0;d214edae7bd2:45797 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=725.51 KB at sequenceid=2022 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1df0fe0a5c2e4fe2bed66e9f57b0fc3c 2024-12-04T14:59:03,125 INFO [M:0;d214edae7bd2:45797 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1df0fe0a5c2e4fe2bed66e9f57b0fc3c 2024-12-04T14:59:03,163 DEBUG [M:0;d214edae7bd2:45797 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b3bb6ccfc6bd4c09ba805ec11c16fc9d is 69, key is d214edae7bd2,34477,1733324158096/rs:state/1733324159713/Put/seqid=0 2024-12-04T14:59:03,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073742413_1589 (size=5156) 2024-12-04T14:59:03,192 INFO [M:0;d214edae7bd2:45797 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2022 (bloomFilter=true), to=hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b3bb6ccfc6bd4c09ba805ec11c16fc9d 2024-12-04T14:59:03,198 DEBUG [M:0;d214edae7bd2:45797 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cfc80be2e9dc4bb5a720e21f101d60d5 as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cfc80be2e9dc4bb5a720e21f101d60d5 2024-12-04T14:59:03,209 INFO [M:0;d214edae7bd2:45797 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cfc80be2e9dc4bb5a720e21f101d60d5, entries=8, sequenceid=2022, filesize=5.5 K 2024-12-04T14:59:03,210 DEBUG [M:0;d214edae7bd2:45797 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1df0fe0a5c2e4fe2bed66e9f57b0fc3c as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1df0fe0a5c2e4fe2bed66e9f57b0fc3c 2024-12-04T14:59:03,221 INFO [M:0;d214edae7bd2:45797 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1df0fe0a5c2e4fe2bed66e9f57b0fc3c 2024-12-04T14:59:03,221 INFO [M:0;d214edae7bd2:45797 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1df0fe0a5c2e4fe2bed66e9f57b0fc3c, entries=164, sequenceid=2022, filesize=42.1 K 2024-12-04T14:59:03,227 DEBUG [M:0;d214edae7bd2:45797 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b3bb6ccfc6bd4c09ba805ec11c16fc9d as hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b3bb6ccfc6bd4c09ba805ec11c16fc9d 2024-12-04T14:59:03,233 INFO [M:0;d214edae7bd2:45797 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41043/user/jenkins/test-data/6dd96ba5-ac62-4b7f-121f-05dd213533f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b3bb6ccfc6bd4c09ba805ec11c16fc9d, entries=1, sequenceid=2022, filesize=5.0 K 2024-12-04T14:59:03,234 INFO [M:0;d214edae7bd2:45797 {}] regionserver.HRegion(3140): Finished flush of dataSize ~726.07 KB/743492, heapSize ~890.21 KB/911576, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 635ms, sequenceid=2022, compaction requested=false 2024-12-04T14:59:03,253 INFO [M:0;d214edae7bd2:45797 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T14:59:03,253 DEBUG [M:0;d214edae7bd2:45797 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733324342599Disabling compacts and flushes for region at 1733324342599Disabling writes for close at 1733324342599Obtaining lock to block concurrent updates at 1733324342599Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733324342599Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=743492, getHeapSize=911816, getOffHeapSize=0, getCellsCount=2211 at 1733324342600 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733324342601 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733324342602 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733324342625 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733324342625Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733324343037 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733324343090 (+53 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733324343091 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733324343125 (+34 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733324343161 (+36 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733324343161Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1879b80a: reopening flushed file at 1733324343197 (+36 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e1dbd25: reopening flushed file at 1733324343209 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3339ea5c: reopening flushed file at 1733324343221 (+12 ms)Finished flush of dataSize ~726.07 KB/743492, heapSize ~890.21 KB/911576, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 635ms, sequenceid=2022, compaction requested=false at 1733324343234 (+13 ms)Writing region close event to WAL at 1733324343253 (+19 ms)Closed at 1733324343253 2024-12-04T14:59:03,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35153 is added to blk_1073741830_1006 (size=876097) 2024-12-04T14:59:03,256 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T14:59:03,259 INFO [M:0;d214edae7bd2:45797 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T14:59:03,259 INFO [M:0;d214edae7bd2:45797 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45797 2024-12-04T14:59:03,260 INFO [M:0;d214edae7bd2:45797 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T14:59:03,401 INFO [M:0;d214edae7bd2:45797 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T14:59:03,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:59:03,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45797-0x101a22151460000, quorum=127.0.0.1:63984, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T14:59:03,406 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T14:59:03,408 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:59:03,408 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:59:03,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:59:03,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/hadoop.log.dir/,STOPPED} 2024-12-04T14:59:03,412 WARN [BP-1812510059-172.17.0.2-1733324154111 heartbeating to localhost/127.0.0.1:41043 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T14:59:03,412 WARN [BP-1812510059-172.17.0.2-1733324154111 heartbeating to localhost/127.0.0.1:41043 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1812510059-172.17.0.2-1733324154111 (Datanode Uuid d3ec1071-8fa2-49ad-85bb-9771668d1f80) service to localhost/127.0.0.1:41043 2024-12-04T14:59:03,413 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/cluster_654e7680-3d13-2f9f-e580-aa09589746a6/data/data1/current/BP-1812510059-172.17.0.2-1733324154111 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:59:03,413 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/cluster_654e7680-3d13-2f9f-e580-aa09589746a6/data/data2/current/BP-1812510059-172.17.0.2-1733324154111 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T14:59:03,414 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T14:59:03,414 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T14:59:03,415 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T14:59:03,451 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T14:59:03,452 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T14:59:03,452 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T14:59:03,452 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T14:59:03,452 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9c28ef-a87e-2091-20aa-b030fcbf4fd7/hadoop.log.dir/,STOPPED} 2024-12-04T14:59:03,480 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T14:59:03,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down